2023-09-06 20:31:19,448 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 2023-09-06 20:31:19,464 main DEBUG Took 0.012835 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-09-06 20:31:19,464 main DEBUG PluginManager 'Core' found 129 plugins 2023-09-06 20:31:19,465 main DEBUG PluginManager 'Level' found 0 plugins 2023-09-06 20:31:19,466 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-09-06 20:31:19,467 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:31:19,475 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-09-06 20:31:19,489 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:31:19,490 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:31:19,491 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:31:19,491 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:31:19,491 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:31:19,492 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:31:19,492 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:31:19,493 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:31:19,493 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:31:19,493 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:31:19,494 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:31:19,494 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:31:19,494 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:31:19,495 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:31:19,495 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:31:19,495 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:31:19,496 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:31:19,496 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:31:19,496 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:31:19,496 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:31:19,497 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:31:19,497 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:31:19,497 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:31:19,498 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:31:19,498 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:31:19,498 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-09-06 20:31:19,500 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:31:19,502 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-09-06 20:31:19,503 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-09-06 20:31:19,504 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-09-06 20:31:19,505 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-09-06 20:31:19,506 main DEBUG PluginManager 'Converter' found 47 plugins 2023-09-06 20:31:19,518 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-09-06 20:31:19,521 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-09-06 20:31:19,522 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-09-06 20:31:19,522 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-09-06 20:31:19,523 main DEBUG createAppenders(={Console}) 2023-09-06 20:31:19,523 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 initialized 2023-09-06 20:31:19,524 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 2023-09-06 20:31:19,524 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 OK. 2023-09-06 20:31:19,525 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-09-06 20:31:19,525 main DEBUG OutputStream closed 2023-09-06 20:31:19,525 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-09-06 20:31:19,525 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-09-06 20:31:19,526 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@2145b572 OK 2023-09-06 20:31:19,570 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-09-06 20:31:19,572 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-09-06 20:31:19,573 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-09-06 20:31:19,574 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-09-06 20:31:19,575 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-09-06 20:31:19,575 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-09-06 20:31:19,575 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-09-06 20:31:19,576 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-09-06 20:31:19,576 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-09-06 20:31:19,576 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-09-06 20:31:19,576 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-09-06 20:31:19,577 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-09-06 20:31:19,577 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-09-06 20:31:19,577 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-09-06 20:31:19,578 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-09-06 20:31:19,578 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-09-06 20:31:19,578 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-09-06 20:31:19,580 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-09-06 20:31:19,583 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-09-06 20:31:19,583 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@49912c99) with optional ClassLoader: null 2023-09-06 20:31:19,583 main DEBUG Shutdown hook enabled. Registering a new one. 2023-09-06 20:31:19,584 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@49912c99] started OK. 2023-09-06T20:31:19,617 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestFullRestore timeout: 13 mins 2023-09-06 20:31:19,622 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-09-06 20:31:19,622 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-09-06T20:31:20,008 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4 2023-09-06T20:31:20,014 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-09-06T20:31:20,016 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:31:20,018 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-09-06T20:31:20,021 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/cluster_26d71070-e599-b36c-2d51-b2ad8bb6051e, deleteOnExit=true 2023-09-06T20:31:20,022 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-09-06T20:31:20,022 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/test.cache.data in system properties and HBase conf 2023-09-06T20:31:20,023 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop.tmp.dir in system properties and HBase conf 2023-09-06T20:31:20,023 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop.log.dir in system properties and HBase conf 2023-09-06T20:31:20,024 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir in system properties and HBase conf 2023-09-06T20:31:20,024 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-09-06T20:31:20,024 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-09-06T20:31:20,140 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-09-06T20:31:20,607 DEBUG [Time-limited test {}] fs.HFileSystem(308): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-09-06T20:31:20,612 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-09-06T20:31:20,612 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-09-06T20:31:20,612 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-09-06T20:31:20,612 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-06T20:31:20,613 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-09-06T20:31:20,613 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-09-06T20:31:20,613 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-06T20:31:20,614 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-06T20:31:20,614 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-09-06T20:31:20,614 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/nfs.dump.dir in system properties and HBase conf 2023-09-06T20:31:20,615 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/java.io.tmpdir in system properties and HBase conf 2023-09-06T20:31:20,615 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-06T20:31:20,615 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-09-06T20:31:20,615 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-09-06T20:31:21,075 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-09-06T20:31:21,079 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-09-06T20:31:21,380 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-09-06T20:31:21,561 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@1b83cad1 via org.mortbay.log.Slf4jLog 2023-09-06T20:31:21,585 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:31:21,644 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:31:21,699 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/java.io.tmpdir/Jetty_localhost_35103_hdfs____bptmgs/webapp 2023-09-06T20:31:21,851 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:35103 2023-09-06T20:31:21,880 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-09-06T20:31:21,880 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-09-06T20:31:22,434 WARN [Listener at localhost/39327 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-09-06T20:31:22,497 WARN [Listener at localhost/39327 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-09-06T20:31:22,516 WARN [Listener at localhost/39327 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:31:22,522 INFO [Listener at localhost/39327 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:31:22,528 INFO [Listener at localhost/39327 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/java.io.tmpdir/Jetty_localhost_46073_datanode____vggvu9/webapp 2023-09-06T20:31:22,635 INFO [Listener at localhost/39327 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:46073 2023-09-06T20:31:22,967 WARN [Listener at localhost/38215 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-09-06T20:31:23,267 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-09-06T20:31:23,424 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x168934711312a654: Processing first storage report for DS-1d0182ba-524c-449c-b659-7241cc36c52e from datanode 1d2bcda5-678a-442c-9e83-5e69f50490fe 2023-09-06T20:31:23,425 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x168934711312a654: from storage DS-1d0182ba-524c-449c-b659-7241cc36c52e node DatanodeRegistration(127.0.0.1:41603, datanodeUuid=1d2bcda5-678a-442c-9e83-5e69f50490fe, infoPort=32963, infoSecurePort=0, ipcPort=38215, storageInfo=lv=-57;cid=testClusterID;nsid=274799815;c=1694032281151), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2023-09-06T20:31:23,425 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x168934711312a654: Processing first storage report for DS-14d35c25-c691-43d4-8c08-dab13a1579fd from datanode 1d2bcda5-678a-442c-9e83-5e69f50490fe 2023-09-06T20:31:23,425 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x168934711312a654: from storage DS-14d35c25-c691-43d4-8c08-dab13a1579fd node DatanodeRegistration(127.0.0.1:41603, datanodeUuid=1d2bcda5-678a-442c-9e83-5e69f50490fe, infoPort=32963, infoSecurePort=0, ipcPort=38215, storageInfo=lv=-57;cid=testClusterID;nsid=274799815;c=1694032281151), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-09-06T20:31:23,495 DEBUG [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4 2023-09-06T20:31:23,611 INFO [Listener at localhost/38215 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/cluster_26d71070-e599-b36c-2d51-b2ad8bb6051e/zookeeper_0, clientPort=53515, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/cluster_26d71070-e599-b36c-2d51-b2ad8bb6051e/zookeeper_0/version-2, dataDirSize=424 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/cluster_26d71070-e599-b36c-2d51-b2ad8bb6051e/zookeeper_0/version-2, dataLogSize=424 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, serverId=0 2023-09-06T20:31:23,634 INFO [Listener at localhost/38215 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=53515 2023-09-06T20:31:23,642 INFO [Listener at localhost/38215 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:31:23,644 INFO [Listener at localhost/38215 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:31:24,326 INFO [Listener at localhost/38215 {}] util.FSUtils(461): Created version file at hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 with version=8 2023-09-06T20:31:24,326 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging 2023-09-06T20:31:24,344 DEBUG [Listener at localhost/38215 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-09-06T20:31:24,344 DEBUG [Listener at localhost/38215 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-09-06T20:31:24,344 DEBUG [Listener at localhost/38215 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-09-06T20:31:24,344 DEBUG [Listener at localhost/38215 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-09-06T20:31:24,564 DEBUG [Listener at localhost/38215 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-09-06T20:31:25,391 INFO [Listener at localhost/38215 {}] client.ConnectionUtils(128): master/jenkins-hbase3:0 server-side Connection retries=45 2023-09-06T20:31:25,446 INFO [Listener at localhost/38215 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-06T20:31:25,447 INFO [Listener at localhost/38215 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-09-06T20:31:25,447 INFO [Listener at localhost/38215 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-09-06T20:31:25,447 INFO [Listener at localhost/38215 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-06T20:31:25,447 INFO [Listener at localhost/38215 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-09-06T20:31:25,617 INFO [Listener at localhost/38215 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-09-06T20:31:25,753 INFO [Listener at localhost/38215 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-09-06T20:31:25,772 DEBUG [Listener at localhost/38215 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-09-06T20:31:25,779 INFO [Listener at localhost/38215 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-09-06T20:31:25,830 DEBUG [Listener at localhost/38215 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 24519 (auto-detected) 2023-09-06T20:31:25,832 DEBUG [Listener at localhost/38215 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:1c:ff:fe:e2:13:ea (auto-detected) 2023-09-06T20:31:25,872 INFO [Listener at localhost/38215 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:37419 2023-09-06T20:31:25,887 INFO [Listener at localhost/38215 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:31:25,889 INFO [Listener at localhost/38215 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:31:25,912 INFO [Listener at localhost/38215 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:37419 connecting to ZooKeeper ensemble=127.0.0.1:53515 2023-09-06T20:31:25,964 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:374190x0, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-09-06T20:31:25,968 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:37419-0x100eefaa8100000 connected 2023-09-06T20:31:26,001 DEBUG [Listener at localhost/38215 {}] zookeeper.ZKUtil(113): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-09-06T20:31:26,005 DEBUG [Listener at localhost/38215 {}] zookeeper.ZKUtil(113): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-09-06T20:31:26,009 DEBUG [Listener at localhost/38215 {}] zookeeper.ZKUtil(113): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-09-06T20:31:26,021 DEBUG [Listener at localhost/38215 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37419 2023-09-06T20:31:26,021 DEBUG [Listener at localhost/38215 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37419 2023-09-06T20:31:26,022 DEBUG [Listener at localhost/38215 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37419 2023-09-06T20:31:26,022 DEBUG [Listener at localhost/38215 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37419 2023-09-06T20:31:26,024 DEBUG [Listener at localhost/38215 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37419 2023-09-06T20:31:26,082 INFO [Listener at localhost/38215 {}] log.Log(170): Logging initialized @7365ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-09-06T20:31:26,223 INFO [Listener at localhost/38215 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-09-06T20:31:26,224 INFO [Listener at localhost/38215 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-09-06T20:31:26,225 INFO [Listener at localhost/38215 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-09-06T20:31:26,228 INFO [Listener at localhost/38215 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-09-06T20:31:26,229 INFO [Listener at localhost/38215 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-09-06T20:31:26,229 INFO [Listener at localhost/38215 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-09-06T20:31:26,233 INFO [Listener at localhost/38215 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-09-06T20:31:26,389 INFO [Listener at localhost/38215 {}] http.HttpServer(1219): Jetty bound to port 41117 2023-09-06T20:31:26,392 INFO [Listener at localhost/38215 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-09-06T20:31:26,438 INFO [Listener at localhost/38215 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:31:26,442 INFO [Listener at localhost/38215 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@55008ff6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop.log.dir/,AVAILABLE} 2023-09-06T20:31:26,443 INFO [Listener at localhost/38215 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:31:26,443 INFO [Listener at localhost/38215 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@7c032c8b{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-09-06T20:31:26,522 INFO [Listener at localhost/38215 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-09-06T20:31:26,536 INFO [Listener at localhost/38215 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-09-06T20:31:26,536 INFO [Listener at localhost/38215 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-09-06T20:31:26,538 INFO [Listener at localhost/38215 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-09-06T20:31:26,546 INFO [Listener at localhost/38215 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:31:27,829 INFO [Listener at localhost/38215 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@1f3fecf1{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-09-06T20:31:27,841 INFO [Listener at localhost/38215 {}] server.AbstractConnector(333): Started ServerConnector@6c0e7723{HTTP/1.1, (http/1.1)}{0.0.0.0:41117} 2023-09-06T20:31:27,841 INFO [Listener at localhost/38215 {}] server.Server(415): Started @9124ms 2023-09-06T20:31:27,845 INFO [Listener at localhost/38215 {}] master.HMaster(485): hbase.rootdir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1, hbase.cluster.distributed=false 2023-09-06T20:31:27,908 INFO [Listener at localhost/38215 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase3:0 server-side Connection retries=45 2023-09-06T20:31:27,908 INFO [Listener at localhost/38215 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-06T20:31:27,909 INFO [Listener at localhost/38215 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-09-06T20:31:27,909 INFO [Listener at localhost/38215 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-09-06T20:31:27,909 INFO [Listener at localhost/38215 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-06T20:31:27,909 INFO [Listener at localhost/38215 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-09-06T20:31:27,917 INFO [Listener at localhost/38215 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-09-06T20:31:27,919 INFO [Listener at localhost/38215 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-09-06T20:31:27,921 INFO [Listener at localhost/38215 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:36961 2023-09-06T20:31:27,923 INFO [Listener at localhost/38215 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-09-06T20:31:27,931 DEBUG [Listener at localhost/38215 {}] mob.MobFileCache(120): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-09-06T20:31:27,932 INFO [Listener at localhost/38215 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:31:27,935 INFO [Listener at localhost/38215 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:31:27,937 INFO [Listener at localhost/38215 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:36961 connecting to ZooKeeper ensemble=127.0.0.1:53515 2023-09-06T20:31:27,941 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:369610x0, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-09-06T20:31:27,942 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:36961-0x100eefaa8100001 connected 2023-09-06T20:31:27,942 DEBUG [Listener at localhost/38215 {}] zookeeper.ZKUtil(113): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-09-06T20:31:27,943 DEBUG [Listener at localhost/38215 {}] zookeeper.ZKUtil(113): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-09-06T20:31:27,944 DEBUG [Listener at localhost/38215 {}] zookeeper.ZKUtil(113): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-09-06T20:31:27,945 DEBUG [Listener at localhost/38215 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36961 2023-09-06T20:31:27,945 DEBUG [Listener at localhost/38215 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36961 2023-09-06T20:31:27,945 DEBUG [Listener at localhost/38215 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36961 2023-09-06T20:31:27,946 DEBUG [Listener at localhost/38215 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36961 2023-09-06T20:31:27,946 DEBUG [Listener at localhost/38215 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36961 2023-09-06T20:31:27,949 INFO [Listener at localhost/38215 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-09-06T20:31:27,949 INFO [Listener at localhost/38215 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-09-06T20:31:27,950 INFO [Listener at localhost/38215 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-09-06T20:31:27,950 INFO [Listener at localhost/38215 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-09-06T20:31:27,951 INFO [Listener at localhost/38215 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-09-06T20:31:27,951 INFO [Listener at localhost/38215 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-09-06T20:31:27,951 INFO [Listener at localhost/38215 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-09-06T20:31:27,953 INFO [Listener at localhost/38215 {}] http.HttpServer(1219): Jetty bound to port 46593 2023-09-06T20:31:27,953 INFO [Listener at localhost/38215 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-09-06T20:31:27,954 INFO [Listener at localhost/38215 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:31:27,955 INFO [Listener at localhost/38215 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@63388c8d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop.log.dir/,AVAILABLE} 2023-09-06T20:31:27,955 INFO [Listener at localhost/38215 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:31:27,955 INFO [Listener at localhost/38215 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@3a910b99{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-09-06T20:31:27,968 INFO [Listener at localhost/38215 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-09-06T20:31:27,969 INFO [Listener at localhost/38215 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-09-06T20:31:27,969 INFO [Listener at localhost/38215 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-09-06T20:31:27,969 INFO [Listener at localhost/38215 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-09-06T20:31:27,970 INFO [Listener at localhost/38215 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:31:27,975 INFO [Listener at localhost/38215 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@fbc47a8{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-09-06T20:31:27,976 INFO [Listener at localhost/38215 {}] server.AbstractConnector(333): Started ServerConnector@324c5f54{HTTP/1.1, (http/1.1)}{0.0.0.0:46593} 2023-09-06T20:31:27,976 INFO [Listener at localhost/38215 {}] server.Server(415): Started @9259ms 2023-09-06T20:31:27,981 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-09-06T20:31:27,984 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@584f5459{HTTP/1.1, (http/1.1)}{0.0.0.0:39713} 2023-09-06T20:31:27,984 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @9267ms 2023-09-06T20:31:27,984 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2388): Adding backup master ZNode /1/backup-masters/jenkins-hbase3.apache.org,37419,1694032284546 2023-09-06T20:31:27,997 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-09-06T20:31:27,997 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-09-06T20:31:28,000 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase3.apache.org,37419,1694032284546 2023-09-06T20:31:28,011 DEBUG [M:0;jenkins-hbase3:37419 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:37419 2023-09-06T20:31:28,021 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-09-06T20:31:28,021 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-09-06T20:31:28,021 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:31:28,021 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:31:28,022 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on existing znode=/1/master 2023-09-06T20:31:28,024 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase3.apache.org,37419,1694032284546 from backup master directory 2023-09-06T20:31:28,024 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on existing znode=/1/master 2023-09-06T20:31:28,030 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase3.apache.org,37419,1694032284546 2023-09-06T20:31:28,031 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-09-06T20:31:28,031 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-09-06T20:31:28,032 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-09-06T20:31:28,032 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,37419,1694032284546 2023-09-06T20:31:28,035 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-09-06T20:31:28,038 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-09-06T20:31:28,526 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(610): Created cluster ID file at hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase.id with ID: 144e8c69-da03-4514-b8c8-e87430dfac85 2023-09-06T20:31:28,571 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:31:28,587 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:31:28,587 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:31:28,632 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:31:28,634 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-09-06T20:31:28,654 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(270): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-09-06T20:31:28,654 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(276): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-09-06T20:31:28,656 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(285): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:283) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:310) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:31:28,697 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:31:28,698 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-09-06T20:31:28,771 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/MasterData/data/master/store 2023-09-06T20:31:28,795 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-09-06T20:31:28,796 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:31:28,796 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-09-06T20:31:28,796 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:31:28,796 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:31:28,797 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-09-06T20:31:28,797 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:31:28,797 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:31:28,797 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-09-06T20:31:28,799 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/MasterData/data/master/store/.initializing 2023-09-06T20:31:28,799 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/MasterData/WALs/jenkins-hbase3.apache.org,37419,1694032284546 2023-09-06T20:31:28,809 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-09-06T20:31:28,823 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C37419%2C1694032284546, suffix=, logDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/MasterData/WALs/jenkins-hbase3.apache.org,37419,1694032284546, archiveDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/MasterData/oldWALs, maxLogs=10 2023-09-06T20:31:28,852 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/MasterData/WALs/jenkins-hbase3.apache.org,37419,1694032284546/jenkins-hbase3.apache.org%2C37419%2C1694032284546.1694032288830, exclude list is [], retry=0 2023-09-06T20:31:28,879 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK] 2023-09-06T20:31:28,886 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:346) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:430) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:424) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:482) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:477) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:489) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:31:28,955 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/MasterData/WALs/jenkins-hbase3.apache.org,37419,1694032284546/jenkins-hbase3.apache.org%2C37419%2C1694032284546.1694032288830 2023-09-06T20:31:28,955 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK]] 2023-09-06T20:31:28,956 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:31:28,956 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:31:28,961 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:31:28,962 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:31:29,032 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:31:29,067 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-09-06T20:31:29,072 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:31:29,077 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:31:29,077 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:31:29,081 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-09-06T20:31:29,081 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:31:29,082 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:31:29,083 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:31:29,085 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-09-06T20:31:29,085 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:31:29,086 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:31:29,087 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:31:29,089 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-09-06T20:31:29,090 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:31:29,091 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:31:29,096 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:31:29,097 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:31:29,115 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-09-06T20:31:29,118 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:31:29,123 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:31:29,124 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9502565760, jitterRate=-0.11500459909439087}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-09-06T20:31:29,130 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-09-06T20:31:29,131 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-09-06T20:31:29,167 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bc0f9ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:31:29,223 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(860): No meta location available on zookeeper, skip migrating... 2023-09-06T20:31:29,243 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-09-06T20:31:29,244 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(563): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-09-06T20:31:29,247 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-09-06T20:31:29,248 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(583): Recovered RegionProcedureStore lease in 1 msec 2023-09-06T20:31:29,255 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(597): Loaded RegionProcedureStore in 6 msec 2023-09-06T20:31:29,255 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-09-06T20:31:29,291 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-09-06T20:31:29,308 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-09-06T20:31:29,311 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-09-06T20:31:29,315 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(150): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-09-06T20:31:29,316 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-09-06T20:31:29,317 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-09-06T20:31:29,321 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-09-06T20:31:29,328 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-09-06T20:31:29,331 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-09-06T20:31:29,332 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-09-06T20:31:29,333 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-09-06T20:31:29,355 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-09-06T20:31:29,356 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-09-06T20:31:29,361 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-09-06T20:31:29,361 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-09-06T20:31:29,361 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:31:29,361 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:31:29,362 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(804): Active/primary master=jenkins-hbase3.apache.org,37419,1694032284546, sessionid=0x100eefaa8100000, setting cluster-up flag (Was=false) 2023-09-06T20:31:29,373 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-09-06T20:31:29,382 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:31:29,382 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:31:29,388 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-09-06T20:31:29,389 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,37419,1694032284546 2023-09-06T20:31:29,393 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:31:29,393 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:31:29,398 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-09-06T20:31:29,400 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,37419,1694032284546 2023-09-06T20:31:29,405 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:31:29,405 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:31:29,410 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-09-06T20:31:29,412 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,37419,1694032284546 2023-09-06T20:31:29,495 DEBUG [RS:0;jenkins-hbase3:36961 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:36961 2023-09-06T20:31:29,496 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(997): ClusterId : 144e8c69-da03-4514-b8c8-e87430dfac85 2023-09-06T20:31:29,498 INFO [RS:0;jenkins-hbase3:36961 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-09-06T20:31:29,501 DEBUG [RS:0;jenkins-hbase3:36961 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-09-06T20:31:29,506 DEBUG [RS:0;jenkins-hbase3:36961 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-09-06T20:31:29,506 DEBUG [RS:0;jenkins-hbase3:36961 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-09-06T20:31:29,512 DEBUG [RS:0;jenkins-hbase3:36961 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-09-06T20:31:29,512 DEBUG [RS:0;jenkins-hbase3:36961 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-09-06T20:31:29,515 DEBUG [RS:0;jenkins-hbase3:36961 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-09-06T20:31:29,516 DEBUG [RS:0;jenkins-hbase3:36961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78f59730, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:31:29,518 DEBUG [RS:0;jenkins-hbase3:36961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ec48328, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-09-06T20:31:29,526 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-09-06T20:31:29,526 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-09-06T20:31:29,526 DEBUG [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-09-06T20:31:29,530 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,37419,1694032284546 with isa=jenkins-hbase3.apache.org/172.31.12.81:36961, startcode=1694032287907 2023-09-06T20:31:29,554 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-09-06T20:31:29,557 DEBUG [RS:0;jenkins-hbase3:36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-09-06T20:31:29,564 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-09-06T20:31:29,569 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-09-06T20:31:29,578 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,37419,1694032284546 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-09-06T20:31:29,583 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-06T20:31:29,583 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-06T20:31:29,583 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-06T20:31:29,583 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-06T20:31:29,584 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-09-06T20:31:29,584 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:31:29,584 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-09-06T20:31:29,585 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:31:29,588 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1694032319588 2023-09-06T20:31:29,591 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-09-06T20:31:29,596 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-09-06T20:31:29,597 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-09-06T20:31:29,598 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-09-06T20:31:29,604 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:31:29,604 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-09-06T20:31:29,609 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-09-06T20:31:29,610 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-09-06T20:31:29,610 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-09-06T20:31:29,611 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-09-06T20:31:29,611 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-09-06T20:31:29,613 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-09-06T20:31:29,618 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-09-06T20:31:29,621 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-09-06T20:31:29,624 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-09-06T20:31:29,633 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-09-06T20:31:29,634 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:31:29,638 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:34637, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-09-06T20:31:29,639 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x57b49bf9 to 127.0.0.1:53515 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:31:29,649 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37419 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3181) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:590) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:31:29,662 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31478b97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:31:29,662 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-09-06T20:31:29,670 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-09-06T20:31:29,670 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-09-06T20:31:29,673 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1694032289673,5,FailOnTimeoutGroup] 2023-09-06T20:31:29,673 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1694032289673,5,FailOnTimeoutGroup] 2023-09-06T20:31:29,673 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-09-06T20:31:29,674 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1635): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-09-06T20:31:29,676 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-09-06T20:31:29,676 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-09-06T20:31:29,694 DEBUG [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(3046): Master is not running yet 2023-09-06T20:31:29,695 WARN [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 100 ms and then retrying. 2023-09-06T20:31:29,796 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,37419,1694032284546 with isa=jenkins-hbase3.apache.org/172.31.12.81:36961, startcode=1694032287907 2023-09-06T20:31:29,801 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37419 {}] master.ServerManager(403): Registering regionserver=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:29,812 DEBUG [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:31:29,812 DEBUG [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:39327 2023-09-06T20:31:29,812 DEBUG [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=41117 2023-09-06T20:31:29,817 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-09-06T20:31:29,818 DEBUG [RS:0;jenkins-hbase3:36961 {}] zookeeper.ZKUtil(111): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:29,818 WARN [RS:0;jenkins-hbase3:36961 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-09-06T20:31:29,818 INFO [RS:0;jenkins-hbase3:36961 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-09-06T20:31:29,818 DEBUG [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:29,821 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,36961,1694032287907] 2023-09-06T20:31:29,836 DEBUG [RS:0;jenkins-hbase3:36961 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-09-06T20:31:29,850 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-09-06T20:31:29,863 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-09-06T20:31:29,867 INFO [RS:0;jenkins-hbase3:36961 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-09-06T20:31:29,867 INFO [RS:0;jenkins-hbase3:36961 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-09-06T20:31:29,868 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-09-06T20:31:29,878 INFO [RS:0;jenkins-hbase3:36961 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-09-06T20:31:29,878 DEBUG [RS:0;jenkins-hbase3:36961 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:31:29,878 DEBUG [RS:0;jenkins-hbase3:36961 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:31:29,879 DEBUG [RS:0;jenkins-hbase3:36961 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:31:29,879 DEBUG [RS:0;jenkins-hbase3:36961 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:31:29,879 DEBUG [RS:0;jenkins-hbase3:36961 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:31:29,879 DEBUG [RS:0;jenkins-hbase3:36961 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-09-06T20:31:29,879 DEBUG [RS:0;jenkins-hbase3:36961 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:31:29,879 DEBUG [RS:0;jenkins-hbase3:36961 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:31:29,879 DEBUG [RS:0;jenkins-hbase3:36961 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:31:29,879 DEBUG [RS:0;jenkins-hbase3:36961 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:31:29,880 DEBUG [RS:0;jenkins-hbase3:36961 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:31:29,880 DEBUG [RS:0;jenkins-hbase3:36961 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-09-06T20:31:29,880 DEBUG [RS:0;jenkins-hbase3:36961 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-09-06T20:31:29,881 INFO [RS:0;jenkins-hbase3:36961 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-09-06T20:31:29,881 INFO [RS:0;jenkins-hbase3:36961 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-09-06T20:31:29,881 INFO [RS:0;jenkins-hbase3:36961 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-09-06T20:31:29,881 INFO [RS:0;jenkins-hbase3:36961 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-09-06T20:31:29,881 INFO [RS:0;jenkins-hbase3:36961 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,36961,1694032287907-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-09-06T20:31:29,897 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-09-06T20:31:29,899 INFO [RS:0;jenkins-hbase3:36961 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,36961,1694032287907-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-09-06T20:31:29,915 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,36961,1694032287907 started 2023-09-06T20:31:29,915 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,36961,1694032287907, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:36961, sessionid=0x100eefaa8100001 2023-09-06T20:31:29,915 DEBUG [RS:0;jenkins-hbase3:36961 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-09-06T20:31:29,916 DEBUG [RS:0;jenkins-hbase3:36961 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:29,916 DEBUG [RS:0;jenkins-hbase3:36961 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,36961,1694032287907' 2023-09-06T20:31:29,916 DEBUG [RS:0;jenkins-hbase3:36961 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-09-06T20:31:29,916 DEBUG [RS:0;jenkins-hbase3:36961 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-09-06T20:31:29,917 DEBUG [RS:0;jenkins-hbase3:36961 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-09-06T20:31:29,917 DEBUG [RS:0;jenkins-hbase3:36961 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-09-06T20:31:29,917 DEBUG [RS:0;jenkins-hbase3:36961 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,36961,1694032287907' 2023-09-06T20:31:29,917 DEBUG [RS:0;jenkins-hbase3:36961 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:31:29,918 DEBUG [RS:0;jenkins-hbase3:36961 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:31:29,918 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-09-06T20:31:29,918 DEBUG [RS:0;jenkins-hbase3:36961 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-09-06T20:31:29,918 DEBUG [RS:0;jenkins-hbase3:36961 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-09-06T20:31:29,919 DEBUG [RS:0;jenkins-hbase3:36961 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:29,919 DEBUG [RS:0;jenkins-hbase3:36961 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,36961,1694032287907' 2023-09-06T20:31:29,919 DEBUG [RS:0;jenkins-hbase3:36961 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-09-06T20:31:29,919 DEBUG [RS:0;jenkins-hbase3:36961 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-09-06T20:31:29,920 DEBUG [RS:0;jenkins-hbase3:36961 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-09-06T20:31:29,920 INFO [RS:0;jenkins-hbase3:36961 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-09-06T20:31:29,920 INFO [RS:0;jenkins-hbase3:36961 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-09-06T20:31:30,030 INFO [RS:0;jenkins-hbase3:36961 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-09-06T20:31:30,033 INFO [RS:0;jenkins-hbase3:36961 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C36961%2C1694032287907, suffix=, logDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907, archiveDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/oldWALs, maxLogs=32 2023-09-06T20:31:30,046 DEBUG [RS:0;jenkins-hbase3:36961 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032290035, exclude list is [], retry=0 2023-09-06T20:31:30,051 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK] 2023-09-06T20:31:30,056 INFO [RS:0;jenkins-hbase3:36961 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032290035 2023-09-06T20:31:30,056 DEBUG [RS:0;jenkins-hbase3:36961 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK]] 2023-09-06T20:31:30,069 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:31:30,072 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-09-06T20:31:30,074 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-09-06T20:31:30,074 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:31:30,075 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:31:30,075 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-09-06T20:31:30,077 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-09-06T20:31:30,077 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:31:30,078 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:31:30,078 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-09-06T20:31:30,080 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-09-06T20:31:30,081 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:31:30,081 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:31:30,083 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/hbase/meta/1588230740 2023-09-06T20:31:30,084 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/hbase/meta/1588230740 2023-09-06T20:31:30,086 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-09-06T20:31:30,089 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-09-06T20:31:30,092 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:31:30,093 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10974324800, jitterRate=0.022063642740249634}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-09-06T20:31:30,095 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-09-06T20:31:30,096 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-09-06T20:31:30,096 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-09-06T20:31:30,096 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-09-06T20:31:30,096 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-09-06T20:31:30,096 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-09-06T20:31:30,098 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-09-06T20:31:30,098 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-09-06T20:31:30,101 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-09-06T20:31:30,101 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-09-06T20:31:30,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-09-06T20:31:30,122 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-09-06T20:31:30,125 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-09-06T20:31:30,277 DEBUG [jenkins-hbase3:37419 {}] assignment.AssignmentManager(2303): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-09-06T20:31:30,289 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:30,298 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,36961,1694032287907, state=OPENING 2023-09-06T20:31:30,308 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-09-06T20:31:30,312 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:31:30,312 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:31:30,313 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-09-06T20:31:30,313 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-09-06T20:31:30,316 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:31:30,510 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(721): New admin connection to jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:30,512 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-09-06T20:31:30,515 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:47092, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-09-06T20:31:30,528 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-09-06T20:31:30,528 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-09-06T20:31:30,529 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-09-06T20:31:30,532 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta, suffix=.meta, logDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907, archiveDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/oldWALs, maxLogs=32 2023-09-06T20:31:30,549 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032290534.meta, exclude list is [], retry=0 2023-09-06T20:31:30,553 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK] 2023-09-06T20:31:30,557 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032290534.meta 2023-09-06T20:31:30,557 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK]] 2023-09-06T20:31:30,558 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:31:30,563 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:31:30,563 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-09-06T20:31:30,580 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-09-06T20:31:30,580 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-09-06T20:31:30,585 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-09-06T20:31:30,586 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:31:30,586 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-09-06T20:31:30,586 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-09-06T20:31:30,589 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-09-06T20:31:30,590 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-09-06T20:31:30,590 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:31:30,591 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:31:30,591 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-09-06T20:31:30,592 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-09-06T20:31:30,592 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:31:30,593 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:31:30,593 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-09-06T20:31:30,594 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-09-06T20:31:30,595 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:31:30,595 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:31:30,597 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/hbase/meta/1588230740 2023-09-06T20:31:30,599 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/hbase/meta/1588230740 2023-09-06T20:31:30,602 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-09-06T20:31:30,605 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-09-06T20:31:30,607 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11114875040, jitterRate=0.035153403878211975}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-09-06T20:31:30,608 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-09-06T20:31:30,619 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1694032290500 2023-09-06T20:31:30,638 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:meta,,1.1588230740 2023-09-06T20:31:30,638 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-09-06T20:31:30,639 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:30,641 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,36961,1694032287907, state=OPEN 2023-09-06T20:31:30,644 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-09-06T20:31:30,644 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-09-06T20:31:30,644 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-09-06T20:31:30,644 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-09-06T20:31:30,650 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=3, resume processing ppid=2 2023-09-06T20:31:30,650 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,36961,1694032287907 in 328 msec 2023-09-06T20:31:30,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=2, resume processing ppid=1 2023-09-06T20:31:30,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 542 msec 2023-09-06T20:31:30,793 INFO [AsyncFSWAL-0-hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/MasterData {}] wal.AbstractFSWAL(1174): Slow sync cost: 130 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK]] 2023-09-06T20:31:30,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.1890 sec 2023-09-06T20:31:30,796 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1063): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1694032290796, completionTime=-1 2023-09-06T20:31:30,796 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(830): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-09-06T20:31:30,796 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1629): Joining cluster... 2023-09-06T20:31:30,863 DEBUG [hconnection-0x77470b9a-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:31:30,867 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:47106, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:31:30,882 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1641): Number of RegionServers=1 2023-09-06T20:31:30,882 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1694032350882 2023-09-06T20:31:30,882 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1694032410882 2023-09-06T20:31:30,882 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1648): Joined the cluster in 85 msec 2023-09-06T20:31:30,905 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37419,1694032284546-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-09-06T20:31:30,905 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37419,1694032284546-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-09-06T20:31:30,905 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37419,1694032284546-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-09-06T20:31:30,907 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:37419, period=300000, unit=MILLISECONDS is enabled. 2023-09-06T20:31:30,907 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-09-06T20:31:30,913 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-09-06T20:31:30,928 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-09-06T20:31:30,929 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2368): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-09-06T20:31:30,939 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-09-06T20:31:30,943 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:31:30,945 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:31:30,946 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:31:31,366 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 72ae4de93b6a56d1410c6ff0eb3e6243, NAME => 'hbase:namespace,,1694032290929.72ae4de93b6a56d1410c6ff0eb3e6243.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:31:31,777 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1694032290929.72ae4de93b6a56d1410c6ff0eb3e6243.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:31:31,778 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing 72ae4de93b6a56d1410c6ff0eb3e6243, disabling compactions & flushes 2023-09-06T20:31:31,778 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1694032290929.72ae4de93b6a56d1410c6ff0eb3e6243. 2023-09-06T20:31:31,778 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1694032290929.72ae4de93b6a56d1410c6ff0eb3e6243. 2023-09-06T20:31:31,778 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1694032290929.72ae4de93b6a56d1410c6ff0eb3e6243. after waiting 0 ms 2023-09-06T20:31:31,778 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1694032290929.72ae4de93b6a56d1410c6ff0eb3e6243. 2023-09-06T20:31:31,778 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1694032290929.72ae4de93b6a56d1410c6ff0eb3e6243. 2023-09-06T20:31:31,778 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for 72ae4de93b6a56d1410c6ff0eb3e6243: 2023-09-06T20:31:31,780 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:31:31,794 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1694032290929.72ae4de93b6a56d1410c6ff0eb3e6243.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1694032291782"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032291782"}]},"ts":"1694032291782"} 2023-09-06T20:31:31,826 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:31:31,828 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:31:31,833 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032291828"}]},"ts":"1694032291828"} 2023-09-06T20:31:31,837 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-09-06T20:31:31,846 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=72ae4de93b6a56d1410c6ff0eb3e6243, ASSIGN}] 2023-09-06T20:31:31,848 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=72ae4de93b6a56d1410c6ff0eb3e6243, ASSIGN 2023-09-06T20:31:31,850 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=72ae4de93b6a56d1410c6ff0eb3e6243, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36961,1694032287907; forceNewPlan=false, retain=false 2023-09-06T20:31:32,001 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=72ae4de93b6a56d1410c6ff0eb3e6243, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:32,005 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 72ae4de93b6a56d1410c6ff0eb3e6243, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:31:32,165 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1694032290929.72ae4de93b6a56d1410c6ff0eb3e6243. 2023-09-06T20:31:32,166 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => 72ae4de93b6a56d1410c6ff0eb3e6243, NAME => 'hbase:namespace,,1694032290929.72ae4de93b6a56d1410c6ff0eb3e6243.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:31:32,166 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:31:32,167 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 72ae4de93b6a56d1410c6ff0eb3e6243 2023-09-06T20:31:32,167 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1694032290929.72ae4de93b6a56d1410c6ff0eb3e6243.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:31:32,167 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for 72ae4de93b6a56d1410c6ff0eb3e6243 2023-09-06T20:31:32,167 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for 72ae4de93b6a56d1410c6ff0eb3e6243 2023-09-06T20:31:32,169 INFO [StoreOpener-72ae4de93b6a56d1410c6ff0eb3e6243-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 72ae4de93b6a56d1410c6ff0eb3e6243 2023-09-06T20:31:32,171 INFO [StoreOpener-72ae4de93b6a56d1410c6ff0eb3e6243-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72ae4de93b6a56d1410c6ff0eb3e6243 columnFamilyName info 2023-09-06T20:31:32,171 DEBUG [StoreOpener-72ae4de93b6a56d1410c6ff0eb3e6243-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:31:32,172 INFO [StoreOpener-72ae4de93b6a56d1410c6ff0eb3e6243-1 {}] regionserver.HStore(324): Store=72ae4de93b6a56d1410c6ff0eb3e6243/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:31:32,174 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/hbase/namespace/72ae4de93b6a56d1410c6ff0eb3e6243 2023-09-06T20:31:32,174 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/hbase/namespace/72ae4de93b6a56d1410c6ff0eb3e6243 2023-09-06T20:31:32,177 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for 72ae4de93b6a56d1410c6ff0eb3e6243 2023-09-06T20:31:32,180 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/hbase/namespace/72ae4de93b6a56d1410c6ff0eb3e6243/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:31:32,181 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened 72ae4de93b6a56d1410c6ff0eb3e6243; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9434018880, jitterRate=-0.1213885247707367}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:31:32,182 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for 72ae4de93b6a56d1410c6ff0eb3e6243: 2023-09-06T20:31:32,184 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:namespace,,1694032290929.72ae4de93b6a56d1410c6ff0eb3e6243., pid=6, masterSystemTime=1694032292159 2023-09-06T20:31:32,187 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:namespace,,1694032290929.72ae4de93b6a56d1410c6ff0eb3e6243. 2023-09-06T20:31:32,187 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1694032290929.72ae4de93b6a56d1410c6ff0eb3e6243. 2023-09-06T20:31:32,188 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=72ae4de93b6a56d1410c6ff0eb3e6243, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:32,194 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=6, resume processing ppid=5 2023-09-06T20:31:32,195 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 72ae4de93b6a56d1410c6ff0eb3e6243, server=jenkins-hbase3.apache.org,36961,1694032287907 in 186 msec 2023-09-06T20:31:32,198 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=5, resume processing ppid=4 2023-09-06T20:31:32,198 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=72ae4de93b6a56d1410c6ff0eb3e6243, ASSIGN in 349 msec 2023-09-06T20:31:32,199 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:31:32,199 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032292199"}]},"ts":"1694032292199"} 2023-09-06T20:31:32,202 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-09-06T20:31:32,205 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:31:32,208 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 1.2750 sec 2023-09-06T20:31:32,243 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-09-06T20:31:32,245 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-09-06T20:31:32,245 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:31:32,245 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:31:32,295 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-09-06T20:31:32,313 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:31:32,317 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 30 msec 2023-09-06T20:31:32,331 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-09-06T20:31:32,342 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:31:32,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 14 msec 2023-09-06T20:31:32,358 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-09-06T20:31:32,360 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-09-06T20:31:32,362 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1193): Master has completed initialization 4.330sec 2023-09-06T20:31:32,364 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-09-06T20:31:32,366 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-09-06T20:31:32,367 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-09-06T20:31:32,368 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-09-06T20:31:32,368 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-09-06T20:31:32,370 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37419,1694032284546-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-09-06T20:31:32,370 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37419,1694032284546-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-09-06T20:31:32,383 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1296): Balancer post startup initialization complete, took 0 seconds 2023-09-06T20:31:32,385 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-09-06T20:31:32,400 DEBUG [Listener at localhost/38215 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x349b8c42 to 127.0.0.1:53515 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:31:32,405 DEBUG [Listener at localhost/38215 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26bba8d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:31:32,410 DEBUG [Listener at localhost/38215 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-09-06T20:31:32,410 DEBUG [Listener at localhost/38215 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-09-06T20:31:32,424 DEBUG [hconnection-0x1f412458-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:31:32,434 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:47108, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:31:32,445 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,37419,1694032284546 2023-09-06T20:31:32,445 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-09-06T20:31:32,445 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/test.cache.data in system properties and HBase conf 2023-09-06T20:31:32,445 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop.tmp.dir in system properties and HBase conf 2023-09-06T20:31:32,445 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop.log.dir in system properties and HBase conf 2023-09-06T20:31:32,445 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir in system properties and HBase conf 2023-09-06T20:31:32,445 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-09-06T20:31:32,445 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-09-06T20:31:32,445 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-09-06T20:31:32,445 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-09-06T20:31:32,445 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-09-06T20:31:32,445 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-06T20:31:32,446 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-09-06T20:31:32,446 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-09-06T20:31:32,446 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-06T20:31:32,446 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-06T20:31:32,446 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-09-06T20:31:32,446 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/nfs.dump.dir in system properties and HBase conf 2023-09-06T20:31:32,446 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/java.io.tmpdir in system properties and HBase conf 2023-09-06T20:31:32,446 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-06T20:31:32,446 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-09-06T20:31:32,446 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-09-06T20:31:33,523 WARN [Thread-249 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:31:33,904 INFO [Thread-249 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:31:33,910 INFO [Thread-249 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_44173_jobhistory____.c9frjd/webapp 2023-09-06T20:31:34,083 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-09-06T20:31:34,083 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-09-06T20:31:34,083 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-09-06T20:31:34,087 INFO [Thread-249 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-09-06T20:31:34,172 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:31:34,290 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:31:34,606 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-09-06T20:31:34,642 INFO [Thread-249 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:44173 2023-09-06T20:31:35,017 WARN [Listener at jenkins-hbase3.apache.org/46105 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:31:35,100 WARN [Listener at jenkins-hbase3.apache.org/46105 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-09-06T20:31:35,100 INFO [Listener at jenkins-hbase3.apache.org/46105 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:31:35,110 INFO [Listener at jenkins-hbase3.apache.org/46105 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_38633_cluster____xxjg1e/webapp 2023-09-06T20:31:35,184 INFO [Listener at jenkins-hbase3.apache.org/46105 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-09-06T20:31:35,184 INFO [Listener at jenkins-hbase3.apache.org/46105 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-09-06T20:31:35,184 INFO [Listener at jenkins-hbase3.apache.org/46105 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-09-06T20:31:35,185 INFO [Listener at jenkins-hbase3.apache.org/46105 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-09-06T20:31:35,194 INFO [Listener at jenkins-hbase3.apache.org/46105 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:31:35,434 INFO [Listener at jenkins-hbase3.apache.org/46105 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:31:35,513 INFO [Listener at jenkins-hbase3.apache.org/46105 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:31:35,520 INFO [Listener at jenkins-hbase3.apache.org/46105 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:38633 2023-09-06T20:31:35,886 WARN [Listener at jenkins-hbase3.apache.org/44105 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:31:35,909 WARN [Listener at jenkins-hbase3.apache.org/44105 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-09-06T20:31:35,910 INFO [Listener at jenkins-hbase3.apache.org/44105 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:31:35,916 INFO [Listener at jenkins-hbase3.apache.org/44105 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_43411_node____.s8vass/webapp 2023-09-06T20:31:35,967 INFO [Listener at jenkins-hbase3.apache.org/44105 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-09-06T20:31:35,968 INFO [Listener at jenkins-hbase3.apache.org/44105 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-09-06T20:31:35,968 INFO [Listener at jenkins-hbase3.apache.org/44105 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-09-06T20:31:35,968 INFO [Listener at jenkins-hbase3.apache.org/44105 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-09-06T20:31:35,976 INFO [Listener at jenkins-hbase3.apache.org/44105 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:31:35,997 INFO [Listener at jenkins-hbase3.apache.org/44105 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:31:36,075 INFO [Listener at jenkins-hbase3.apache.org/44105 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:31:36,085 INFO [Listener at jenkins-hbase3.apache.org/44105 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:43411 2023-09-06T20:31:36,123 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:31:36,203 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-09-06T20:31:36,203 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-09-06T20:31:36,205 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-09-06T20:31:36,206 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-09-06T20:31:36,209 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:31:36,209 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-09-06T20:31:36,209 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-09-06T20:31:36,210 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-09-06T20:31:36,210 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-09-06T20:31:36,210 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-09-06T20:31:36,235 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-09-06T20:31:36,236 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-09-06T20:31:36,297 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:31:36,303 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-09-06T20:31:36,304 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:31:36,313 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_37177_node____hiczyk/webapp 2023-09-06T20:31:36,371 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-09-06T20:31:36,371 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-09-06T20:31:36,371 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-09-06T20:31:36,371 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-09-06T20:31:36,381 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:31:36,401 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:31:36,461 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:31:36,467 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:37177 2023-09-06T20:31:36,496 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-09-06T20:31:36,496 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:39327/backupUT 2023-09-06T20:31:36,508 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:31:36,512 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48590, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:31:36,521 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns1'} 2023-09-06T20:31:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-09-06T20:31:36,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-09-06T20:31:36,547 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:31:36,552 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 28 msec 2023-09-06T20:31:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-09-06T20:31:36,644 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns2'} 2023-09-06T20:31:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-09-06T20:31:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=10 2023-09-06T20:31:36,658 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:31:36,666 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 20 msec 2023-09-06T20:31:36,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=10 2023-09-06T20:31:36,754 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns3'} 2023-09-06T20:31:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-09-06T20:31:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=11 2023-09-06T20:31:36,766 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:31:36,771 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 16 msec 2023-09-06T20:31:36,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=11 2023-09-06T20:31:36,864 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns4'} 2023-09-06T20:31:36,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-09-06T20:31:36,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-09-06T20:31:36,877 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:31:36,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 17 msec 2023-09-06T20:31:36,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-09-06T20:31:36,986 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'test-1694032296496', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:31:36,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1694032296496 2023-09-06T20:31:36,991 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1694032296496 execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:31:36,991 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "test-1694032296496" procId is: 13 2023-09-06T20:31:36,992 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:31:36,993 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1694032296496 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:31:36,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-09-06T20:31:37,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-09-06T20:31:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-09-06T20:31:37,425 INFO [RegionOpenAndInit-test-1694032296496-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 98558d33a94bee7957598a35e3cbc370, NAME => 'test-1694032296496,,1694032296981.98558d33a94bee7957598a35e3cbc370.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1694032296496', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:31:37,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-09-06T20:31:37,844 DEBUG [RegionOpenAndInit-test-1694032296496-pool-0 {}] regionserver.HRegion(888): Instantiated test-1694032296496,,1694032296981.98558d33a94bee7957598a35e3cbc370.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:31:37,844 DEBUG [RegionOpenAndInit-test-1694032296496-pool-0 {}] regionserver.HRegion(1668): Closing 98558d33a94bee7957598a35e3cbc370, disabling compactions & flushes 2023-09-06T20:31:37,844 INFO [RegionOpenAndInit-test-1694032296496-pool-0 {}] regionserver.HRegion(1690): Closing region test-1694032296496,,1694032296981.98558d33a94bee7957598a35e3cbc370. 2023-09-06T20:31:37,844 DEBUG [RegionOpenAndInit-test-1694032296496-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1694032296496,,1694032296981.98558d33a94bee7957598a35e3cbc370. 2023-09-06T20:31:37,844 DEBUG [RegionOpenAndInit-test-1694032296496-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1694032296496,,1694032296981.98558d33a94bee7957598a35e3cbc370. after waiting 0 ms 2023-09-06T20:31:37,844 DEBUG [RegionOpenAndInit-test-1694032296496-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1694032296496,,1694032296981.98558d33a94bee7957598a35e3cbc370. 2023-09-06T20:31:37,844 INFO [RegionOpenAndInit-test-1694032296496-pool-0 {}] regionserver.HRegion(1909): Closed test-1694032296496,,1694032296981.98558d33a94bee7957598a35e3cbc370. 2023-09-06T20:31:37,844 DEBUG [RegionOpenAndInit-test-1694032296496-pool-0 {}] regionserver.HRegion(1622): Region close journal for 98558d33a94bee7957598a35e3cbc370: 2023-09-06T20:31:37,846 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1694032296496 execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:31:37,846 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1694032296496,,1694032296981.98558d33a94bee7957598a35e3cbc370.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1694032297846"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032297846"}]},"ts":"1694032297846"} 2023-09-06T20:31:37,849 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:31:37,851 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1694032296496 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:31:37,851 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1694032296496","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032297851"}]},"ts":"1694032297851"} 2023-09-06T20:31:37,853 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1694032296496, state=ENABLING in hbase:meta 2023-09-06T20:31:37,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1694032296496, region=98558d33a94bee7957598a35e3cbc370, ASSIGN}] 2023-09-06T20:31:37,861 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1694032296496, region=98558d33a94bee7957598a35e3cbc370, ASSIGN 2023-09-06T20:31:37,862 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1694032296496, region=98558d33a94bee7957598a35e3cbc370, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36961,1694032287907; forceNewPlan=false, retain=false 2023-09-06T20:31:38,014 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=98558d33a94bee7957598a35e3cbc370, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:38,017 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure 98558d33a94bee7957598a35e3cbc370, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:31:38,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-09-06T20:31:38,179 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1694032296496,,1694032296981.98558d33a94bee7957598a35e3cbc370. 2023-09-06T20:31:38,180 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => 98558d33a94bee7957598a35e3cbc370, NAME => 'test-1694032296496,,1694032296981.98558d33a94bee7957598a35e3cbc370.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:31:38,180 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:31:38,180 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1694032296496 98558d33a94bee7957598a35e3cbc370 2023-09-06T20:31:38,180 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1694032296496,,1694032296981.98558d33a94bee7957598a35e3cbc370.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:31:38,181 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for 98558d33a94bee7957598a35e3cbc370 2023-09-06T20:31:38,181 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for 98558d33a94bee7957598a35e3cbc370 2023-09-06T20:31:38,182 INFO [StoreOpener-98558d33a94bee7957598a35e3cbc370-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 98558d33a94bee7957598a35e3cbc370 2023-09-06T20:31:38,186 INFO [StoreOpener-98558d33a94bee7957598a35e3cbc370-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98558d33a94bee7957598a35e3cbc370 columnFamilyName f 2023-09-06T20:31:38,186 DEBUG [StoreOpener-98558d33a94bee7957598a35e3cbc370-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:31:38,187 INFO [StoreOpener-98558d33a94bee7957598a35e3cbc370-1 {}] regionserver.HStore(324): Store=98558d33a94bee7957598a35e3cbc370/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:31:38,188 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/98558d33a94bee7957598a35e3cbc370 2023-09-06T20:31:38,189 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/98558d33a94bee7957598a35e3cbc370 2023-09-06T20:31:38,193 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for 98558d33a94bee7957598a35e3cbc370 2023-09-06T20:31:38,198 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/98558d33a94bee7957598a35e3cbc370/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:31:38,199 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened 98558d33a94bee7957598a35e3cbc370; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10936160480, jitterRate=0.018509313464164734}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:31:38,200 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for 98558d33a94bee7957598a35e3cbc370: 2023-09-06T20:31:38,201 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1694032296496,,1694032296981.98558d33a94bee7957598a35e3cbc370., pid=15, masterSystemTime=1694032298170 2023-09-06T20:31:38,204 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1694032296496,,1694032296981.98558d33a94bee7957598a35e3cbc370. 2023-09-06T20:31:38,204 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1694032296496,,1694032296981.98558d33a94bee7957598a35e3cbc370. 2023-09-06T20:31:38,205 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=98558d33a94bee7957598a35e3cbc370, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:38,211 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=15, resume processing ppid=14 2023-09-06T20:31:38,211 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure 98558d33a94bee7957598a35e3cbc370, server=jenkins-hbase3.apache.org,36961,1694032287907 in 191 msec 2023-09-06T20:31:38,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=14, resume processing ppid=13 2023-09-06T20:31:38,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1694032296496, region=98558d33a94bee7957598a35e3cbc370, ASSIGN in 353 msec 2023-09-06T20:31:38,215 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1694032296496 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:31:38,215 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1694032296496","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032298215"}]},"ts":"1694032298215"} 2023-09-06T20:31:38,225 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1694032296496, state=ENABLED in hbase:meta 2023-09-06T20:31:38,228 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1694032296496 execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:31:38,230 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1694032296496 in 1.2410 sec 2023-09-06T20:31:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-09-06T20:31:39,108 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:test-1694032296496, procId: 13 completed 2023-09-06T20:31:39,110 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5acb1a4b to 127.0.0.1:53515 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:31:39,115 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c19b199, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:31:39,117 DEBUG [hconnection-0x5a0d4163-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:31:39,120 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:47112, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:31:39,125 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HRegion(8210): writing data to region test-1694032296496,,1694032296981.98558d33a94bee7957598a35e3cbc370. with WAL disabled. Data may be lost in the event of a crash. 2023-09-06T20:31:39,223 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns2:test-16940322964961', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:31:39,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-16940322964961 2023-09-06T20:31:39,227 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16940322964961 execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:31:39,228 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:31:39,228 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "test-16940322964961" procId is: 16 2023-09-06T20:31:39,229 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-16940322964961 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:31:39,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:31:39,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:31:39,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:31:39,647 INFO [RegionOpenAndInit-ns2:test-16940322964961-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 1932330ab1e654ad2ac9ae28fd5fcdb3, NAME => 'ns2:test-16940322964961,,1694032299223.1932330ab1e654ad2ac9ae28fd5fcdb3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16940322964961', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:31:39,670 DEBUG [RegionOpenAndInit-ns2:test-16940322964961-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16940322964961,,1694032299223.1932330ab1e654ad2ac9ae28fd5fcdb3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:31:39,670 DEBUG [RegionOpenAndInit-ns2:test-16940322964961-pool-0 {}] regionserver.HRegion(1668): Closing 1932330ab1e654ad2ac9ae28fd5fcdb3, disabling compactions & flushes 2023-09-06T20:31:39,671 INFO [RegionOpenAndInit-ns2:test-16940322964961-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16940322964961,,1694032299223.1932330ab1e654ad2ac9ae28fd5fcdb3. 2023-09-06T20:31:39,671 DEBUG [RegionOpenAndInit-ns2:test-16940322964961-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16940322964961,,1694032299223.1932330ab1e654ad2ac9ae28fd5fcdb3. 2023-09-06T20:31:39,671 DEBUG [RegionOpenAndInit-ns2:test-16940322964961-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16940322964961,,1694032299223.1932330ab1e654ad2ac9ae28fd5fcdb3. after waiting 0 ms 2023-09-06T20:31:39,671 DEBUG [RegionOpenAndInit-ns2:test-16940322964961-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16940322964961,,1694032299223.1932330ab1e654ad2ac9ae28fd5fcdb3. 2023-09-06T20:31:39,671 INFO [RegionOpenAndInit-ns2:test-16940322964961-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16940322964961,,1694032299223.1932330ab1e654ad2ac9ae28fd5fcdb3. 2023-09-06T20:31:39,671 DEBUG [RegionOpenAndInit-ns2:test-16940322964961-pool-0 {}] regionserver.HRegion(1622): Region close journal for 1932330ab1e654ad2ac9ae28fd5fcdb3: 2023-09-06T20:31:39,673 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-16940322964961 execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:31:39,673 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16940322964961,,1694032299223.1932330ab1e654ad2ac9ae28fd5fcdb3.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1694032299673"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032299673"}]},"ts":"1694032299673"} 2023-09-06T20:31:39,676 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:31:39,678 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-16940322964961 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:31:39,678 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16940322964961","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032299678"}]},"ts":"1694032299678"} 2023-09-06T20:31:39,680 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16940322964961, state=ENABLING in hbase:meta 2023-09-06T20:31:39,685 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16940322964961, region=1932330ab1e654ad2ac9ae28fd5fcdb3, ASSIGN}] 2023-09-06T20:31:39,687 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16940322964961, region=1932330ab1e654ad2ac9ae28fd5fcdb3, ASSIGN 2023-09-06T20:31:39,689 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16940322964961, region=1932330ab1e654ad2ac9ae28fd5fcdb3, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36961,1694032287907; forceNewPlan=false, retain=false 2023-09-06T20:31:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:31:39,840 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=1932330ab1e654ad2ac9ae28fd5fcdb3, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:39,843 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure 1932330ab1e654ad2ac9ae28fd5fcdb3, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:31:40,000 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-16940322964961,,1694032299223.1932330ab1e654ad2ac9ae28fd5fcdb3. 2023-09-06T20:31:40,000 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => 1932330ab1e654ad2ac9ae28fd5fcdb3, NAME => 'ns2:test-16940322964961,,1694032299223.1932330ab1e654ad2ac9ae28fd5fcdb3.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:31:40,000 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:31:40,001 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16940322964961 1932330ab1e654ad2ac9ae28fd5fcdb3 2023-09-06T20:31:40,001 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-16940322964961,,1694032299223.1932330ab1e654ad2ac9ae28fd5fcdb3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:31:40,001 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for 1932330ab1e654ad2ac9ae28fd5fcdb3 2023-09-06T20:31:40,001 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for 1932330ab1e654ad2ac9ae28fd5fcdb3 2023-09-06T20:31:40,003 INFO [StoreOpener-1932330ab1e654ad2ac9ae28fd5fcdb3-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 1932330ab1e654ad2ac9ae28fd5fcdb3 2023-09-06T20:31:40,005 INFO [StoreOpener-1932330ab1e654ad2ac9ae28fd5fcdb3-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1932330ab1e654ad2ac9ae28fd5fcdb3 columnFamilyName f 2023-09-06T20:31:40,005 DEBUG [StoreOpener-1932330ab1e654ad2ac9ae28fd5fcdb3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:31:40,006 INFO [StoreOpener-1932330ab1e654ad2ac9ae28fd5fcdb3-1 {}] regionserver.HStore(324): Store=1932330ab1e654ad2ac9ae28fd5fcdb3/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:31:40,007 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/1932330ab1e654ad2ac9ae28fd5fcdb3 2023-09-06T20:31:40,008 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/1932330ab1e654ad2ac9ae28fd5fcdb3 2023-09-06T20:31:40,011 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for 1932330ab1e654ad2ac9ae28fd5fcdb3 2023-09-06T20:31:40,017 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/1932330ab1e654ad2ac9ae28fd5fcdb3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:31:40,018 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened 1932330ab1e654ad2ac9ae28fd5fcdb3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11406143360, jitterRate=0.06227988004684448}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:31:40,018 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for 1932330ab1e654ad2ac9ae28fd5fcdb3: 2023-09-06T20:31:40,019 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:test-16940322964961,,1694032299223.1932330ab1e654ad2ac9ae28fd5fcdb3., pid=18, masterSystemTime=1694032299995 2023-09-06T20:31:40,022 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:test-16940322964961,,1694032299223.1932330ab1e654ad2ac9ae28fd5fcdb3. 2023-09-06T20:31:40,022 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-16940322964961,,1694032299223.1932330ab1e654ad2ac9ae28fd5fcdb3. 2023-09-06T20:31:40,023 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=1932330ab1e654ad2ac9ae28fd5fcdb3, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:40,028 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=18, resume processing ppid=17 2023-09-06T20:31:40,028 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure 1932330ab1e654ad2ac9ae28fd5fcdb3, server=jenkins-hbase3.apache.org,36961,1694032287907 in 182 msec 2023-09-06T20:31:40,030 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=17, resume processing ppid=16 2023-09-06T20:31:40,030 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16940322964961, region=1932330ab1e654ad2ac9ae28fd5fcdb3, ASSIGN in 343 msec 2023-09-06T20:31:40,031 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-16940322964961 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:31:40,031 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16940322964961","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032300031"}]},"ts":"1694032300031"} 2023-09-06T20:31:40,034 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16940322964961, state=ENABLED in hbase:meta 2023-09-06T20:31:40,037 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16940322964961 execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:31:40,039 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-16940322964961 in 813 msec 2023-09-06T20:31:40,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:31:40,337 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns2:test-16940322964961, procId: 16 completed 2023-09-06T20:31:40,341 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HRegion(8210): writing data to region ns2:test-16940322964961,,1694032299223.1932330ab1e654ad2ac9ae28fd5fcdb3. with WAL disabled. Data may be lost in the event of a crash. 2023-09-06T20:31:40,440 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns3:test-16940322964962', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:31:40,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-16940322964962 2023-09-06T20:31:40,445 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16940322964962 execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:31:40,445 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:31:40,445 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "test-16940322964962" procId is: 19 2023-09-06T20:31:40,446 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-16940322964962 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:31:40,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-09-06T20:31:40,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-09-06T20:31:40,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-09-06T20:31:40,864 INFO [RegionOpenAndInit-ns3:test-16940322964962-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 18ddf30d0a5d8ca482131f8c37827552, NAME => 'ns3:test-16940322964962,,1694032300440.18ddf30d0a5d8ca482131f8c37827552.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16940322964962', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:31:41,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-09-06T20:31:41,282 DEBUG [RegionOpenAndInit-ns3:test-16940322964962-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16940322964962,,1694032300440.18ddf30d0a5d8ca482131f8c37827552.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:31:41,282 DEBUG [RegionOpenAndInit-ns3:test-16940322964962-pool-0 {}] regionserver.HRegion(1668): Closing 18ddf30d0a5d8ca482131f8c37827552, disabling compactions & flushes 2023-09-06T20:31:41,282 INFO [RegionOpenAndInit-ns3:test-16940322964962-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16940322964962,,1694032300440.18ddf30d0a5d8ca482131f8c37827552. 2023-09-06T20:31:41,282 DEBUG [RegionOpenAndInit-ns3:test-16940322964962-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16940322964962,,1694032300440.18ddf30d0a5d8ca482131f8c37827552. 2023-09-06T20:31:41,283 DEBUG [RegionOpenAndInit-ns3:test-16940322964962-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16940322964962,,1694032300440.18ddf30d0a5d8ca482131f8c37827552. after waiting 0 ms 2023-09-06T20:31:41,283 DEBUG [RegionOpenAndInit-ns3:test-16940322964962-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16940322964962,,1694032300440.18ddf30d0a5d8ca482131f8c37827552. 2023-09-06T20:31:41,283 INFO [RegionOpenAndInit-ns3:test-16940322964962-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16940322964962,,1694032300440.18ddf30d0a5d8ca482131f8c37827552. 2023-09-06T20:31:41,283 DEBUG [RegionOpenAndInit-ns3:test-16940322964962-pool-0 {}] regionserver.HRegion(1622): Region close journal for 18ddf30d0a5d8ca482131f8c37827552: 2023-09-06T20:31:41,284 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-16940322964962 execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:31:41,285 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16940322964962,,1694032300440.18ddf30d0a5d8ca482131f8c37827552.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1694032301284"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032301284"}]},"ts":"1694032301284"} 2023-09-06T20:31:41,287 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:31:41,288 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-16940322964962 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:31:41,288 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16940322964962","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032301288"}]},"ts":"1694032301288"} 2023-09-06T20:31:41,290 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16940322964962, state=ENABLING in hbase:meta 2023-09-06T20:31:41,295 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16940322964962, region=18ddf30d0a5d8ca482131f8c37827552, ASSIGN}] 2023-09-06T20:31:41,297 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16940322964962, region=18ddf30d0a5d8ca482131f8c37827552, ASSIGN 2023-09-06T20:31:41,298 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16940322964962, region=18ddf30d0a5d8ca482131f8c37827552, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36961,1694032287907; forceNewPlan=false, retain=false 2023-09-06T20:31:41,449 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=18ddf30d0a5d8ca482131f8c37827552, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:41,452 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure 18ddf30d0a5d8ca482131f8c37827552, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:31:41,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-09-06T20:31:41,610 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-16940322964962,,1694032300440.18ddf30d0a5d8ca482131f8c37827552. 2023-09-06T20:31:41,611 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => 18ddf30d0a5d8ca482131f8c37827552, NAME => 'ns3:test-16940322964962,,1694032300440.18ddf30d0a5d8ca482131f8c37827552.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:31:41,611 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:31:41,611 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16940322964962 18ddf30d0a5d8ca482131f8c37827552 2023-09-06T20:31:41,611 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-16940322964962,,1694032300440.18ddf30d0a5d8ca482131f8c37827552.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:31:41,611 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for 18ddf30d0a5d8ca482131f8c37827552 2023-09-06T20:31:41,611 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for 18ddf30d0a5d8ca482131f8c37827552 2023-09-06T20:31:41,613 INFO [StoreOpener-18ddf30d0a5d8ca482131f8c37827552-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 18ddf30d0a5d8ca482131f8c37827552 2023-09-06T20:31:41,615 INFO [StoreOpener-18ddf30d0a5d8ca482131f8c37827552-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18ddf30d0a5d8ca482131f8c37827552 columnFamilyName f 2023-09-06T20:31:41,615 DEBUG [StoreOpener-18ddf30d0a5d8ca482131f8c37827552-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:31:41,616 INFO [StoreOpener-18ddf30d0a5d8ca482131f8c37827552-1 {}] regionserver.HStore(324): Store=18ddf30d0a5d8ca482131f8c37827552/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:31:41,617 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/test-16940322964962/18ddf30d0a5d8ca482131f8c37827552 2023-09-06T20:31:41,618 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/test-16940322964962/18ddf30d0a5d8ca482131f8c37827552 2023-09-06T20:31:41,621 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for 18ddf30d0a5d8ca482131f8c37827552 2023-09-06T20:31:41,626 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/test-16940322964962/18ddf30d0a5d8ca482131f8c37827552/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:31:41,627 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened 18ddf30d0a5d8ca482131f8c37827552; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11990227200, jitterRate=0.116676926612854}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:31:41,628 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for 18ddf30d0a5d8ca482131f8c37827552: 2023-09-06T20:31:41,629 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:test-16940322964962,,1694032300440.18ddf30d0a5d8ca482131f8c37827552., pid=21, masterSystemTime=1694032301605 2023-09-06T20:31:41,631 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:test-16940322964962,,1694032300440.18ddf30d0a5d8ca482131f8c37827552. 2023-09-06T20:31:41,631 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-16940322964962,,1694032300440.18ddf30d0a5d8ca482131f8c37827552. 2023-09-06T20:31:41,632 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=18ddf30d0a5d8ca482131f8c37827552, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:41,637 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=21, resume processing ppid=20 2023-09-06T20:31:41,637 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure 18ddf30d0a5d8ca482131f8c37827552, server=jenkins-hbase3.apache.org,36961,1694032287907 in 182 msec 2023-09-06T20:31:41,640 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=20, resume processing ppid=19 2023-09-06T20:31:41,640 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16940322964962, region=18ddf30d0a5d8ca482131f8c37827552, ASSIGN in 342 msec 2023-09-06T20:31:41,641 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-16940322964962 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:31:41,641 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16940322964962","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032301641"}]},"ts":"1694032301641"} 2023-09-06T20:31:41,642 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16940322964962, state=ENABLED in hbase:meta 2023-09-06T20:31:41,645 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16940322964962 execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:31:41,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-16940322964962 in 1.2050 sec 2023-09-06T20:31:42,237 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16940322964962' 2023-09-06T20:31:42,238 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1694032296496' 2023-09-06T20:31:42,238 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-16940322964961' 2023-09-06T20:31:42,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-09-06T20:31:42,556 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns3:test-16940322964962, procId: 19 completed 2023-09-06T20:31:42,557 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-16940322964962 get assigned. Timeout = 60000ms 2023-09-06T20:31:42,560 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-06T20:31:42,565 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-16940322964962 assigned to meta. Checking AM states. 2023-09-06T20:31:42,566 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-06T20:31:42,566 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-16940322964962 assigned. 2023-09-06T20:31:42,569 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns4:test-16940322964963', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:31:42,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-16940322964963 2023-09-06T20:31:42,573 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16940322964963 execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:31:42,573 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:31:42,574 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns4" qualifier: "test-16940322964963" procId is: 22 2023-09-06T20:31:42,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:31:42,575 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-16940322964963 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:31:42,595 INFO [RegionOpenAndInit-ns4:test-16940322964963-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 534ff531d20a23c3227d44d6f5ad3251, NAME => 'ns4:test-16940322964963,,1694032302569.534ff531d20a23c3227d44d6f5ad3251.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-16940322964963', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:31:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:31:42,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:31:43,028 DEBUG [RegionOpenAndInit-ns4:test-16940322964963-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-16940322964963,,1694032302569.534ff531d20a23c3227d44d6f5ad3251.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:31:43,028 DEBUG [RegionOpenAndInit-ns4:test-16940322964963-pool-0 {}] regionserver.HRegion(1668): Closing 534ff531d20a23c3227d44d6f5ad3251, disabling compactions & flushes 2023-09-06T20:31:43,028 INFO [RegionOpenAndInit-ns4:test-16940322964963-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-16940322964963,,1694032302569.534ff531d20a23c3227d44d6f5ad3251. 2023-09-06T20:31:43,029 DEBUG [RegionOpenAndInit-ns4:test-16940322964963-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16940322964963,,1694032302569.534ff531d20a23c3227d44d6f5ad3251. 2023-09-06T20:31:43,029 DEBUG [RegionOpenAndInit-ns4:test-16940322964963-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16940322964963,,1694032302569.534ff531d20a23c3227d44d6f5ad3251. after waiting 0 ms 2023-09-06T20:31:43,029 DEBUG [RegionOpenAndInit-ns4:test-16940322964963-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16940322964963,,1694032302569.534ff531d20a23c3227d44d6f5ad3251. 2023-09-06T20:31:43,029 INFO [RegionOpenAndInit-ns4:test-16940322964963-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-16940322964963,,1694032302569.534ff531d20a23c3227d44d6f5ad3251. 2023-09-06T20:31:43,029 DEBUG [RegionOpenAndInit-ns4:test-16940322964963-pool-0 {}] regionserver.HRegion(1622): Region close journal for 534ff531d20a23c3227d44d6f5ad3251: 2023-09-06T20:31:43,030 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-16940322964963 execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:31:43,031 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-16940322964963,,1694032302569.534ff531d20a23c3227d44d6f5ad3251.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1694032303030"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032303030"}]},"ts":"1694032303030"} 2023-09-06T20:31:43,033 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:31:43,034 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-16940322964963 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:31:43,034 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16940322964963","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032303034"}]},"ts":"1694032303034"} 2023-09-06T20:31:43,036 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16940322964963, state=ENABLING in hbase:meta 2023-09-06T20:31:43,041 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16940322964963, region=534ff531d20a23c3227d44d6f5ad3251, ASSIGN}] 2023-09-06T20:31:43,044 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16940322964963, region=534ff531d20a23c3227d44d6f5ad3251, ASSIGN 2023-09-06T20:31:43,045 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-16940322964963, region=534ff531d20a23c3227d44d6f5ad3251, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36961,1694032287907; forceNewPlan=false, retain=false 2023-09-06T20:31:43,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:31:43,196 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=534ff531d20a23c3227d44d6f5ad3251, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:43,198 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure 534ff531d20a23c3227d44d6f5ad3251, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:31:43,355 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-16940322964963,,1694032302569.534ff531d20a23c3227d44d6f5ad3251. 2023-09-06T20:31:43,355 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => 534ff531d20a23c3227d44d6f5ad3251, NAME => 'ns4:test-16940322964963,,1694032302569.534ff531d20a23c3227d44d6f5ad3251.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:31:43,355 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:31:43,355 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16940322964963 534ff531d20a23c3227d44d6f5ad3251 2023-09-06T20:31:43,356 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-16940322964963,,1694032302569.534ff531d20a23c3227d44d6f5ad3251.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:31:43,356 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for 534ff531d20a23c3227d44d6f5ad3251 2023-09-06T20:31:43,356 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for 534ff531d20a23c3227d44d6f5ad3251 2023-09-06T20:31:43,357 INFO [StoreOpener-534ff531d20a23c3227d44d6f5ad3251-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 534ff531d20a23c3227d44d6f5ad3251 2023-09-06T20:31:43,359 INFO [StoreOpener-534ff531d20a23c3227d44d6f5ad3251-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 534ff531d20a23c3227d44d6f5ad3251 columnFamilyName f 2023-09-06T20:31:43,359 DEBUG [StoreOpener-534ff531d20a23c3227d44d6f5ad3251-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:31:43,360 INFO [StoreOpener-534ff531d20a23c3227d44d6f5ad3251-1 {}] regionserver.HStore(324): Store=534ff531d20a23c3227d44d6f5ad3251/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:31:43,361 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns4/test-16940322964963/534ff531d20a23c3227d44d6f5ad3251 2023-09-06T20:31:43,361 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns4/test-16940322964963/534ff531d20a23c3227d44d6f5ad3251 2023-09-06T20:31:43,364 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for 534ff531d20a23c3227d44d6f5ad3251 2023-09-06T20:31:43,367 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns4/test-16940322964963/534ff531d20a23c3227d44d6f5ad3251/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:31:43,368 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened 534ff531d20a23c3227d44d6f5ad3251; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11563253280, jitterRate=0.07691188156604767}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:31:43,368 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for 534ff531d20a23c3227d44d6f5ad3251: 2023-09-06T20:31:43,370 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2556): Post open deploy tasks for ns4:test-16940322964963,,1694032302569.534ff531d20a23c3227d44d6f5ad3251., pid=24, masterSystemTime=1694032303351 2023-09-06T20:31:43,372 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2583): Finished post open deploy task for ns4:test-16940322964963,,1694032302569.534ff531d20a23c3227d44d6f5ad3251. 2023-09-06T20:31:43,372 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-16940322964963,,1694032302569.534ff531d20a23c3227d44d6f5ad3251. 2023-09-06T20:31:43,372 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=534ff531d20a23c3227d44d6f5ad3251, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:43,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=24, resume processing ppid=23 2023-09-06T20:31:43,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure 534ff531d20a23c3227d44d6f5ad3251, server=jenkins-hbase3.apache.org,36961,1694032287907 in 177 msec 2023-09-06T20:31:43,380 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=23, resume processing ppid=22 2023-09-06T20:31:43,380 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-16940322964963, region=534ff531d20a23c3227d44d6f5ad3251, ASSIGN in 336 msec 2023-09-06T20:31:43,381 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-16940322964963 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:31:43,381 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16940322964963","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032303381"}]},"ts":"1694032303381"} 2023-09-06T20:31:43,383 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16940322964963, state=ENABLED in hbase:meta 2023-09-06T20:31:43,386 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16940322964963 execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:31:43,387 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-16940322964963 in 817 msec 2023-09-06T20:31:43,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:31:43,684 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns4:test-16940322964963, procId: 22 completed 2023-09-06T20:31:43,685 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-16940322964963 get assigned. Timeout = 60000ms 2023-09-06T20:31:43,685 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-06T20:31:43,688 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:31:43,689 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-16940322964963 assigned to meta. Checking AM states. 2023-09-06T20:31:43,689 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-06T20:31:43,690 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-16940322964963 assigned. 2023-09-06T20:31:43,690 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5acb1a4b to 127.0.0.1:53515 2023-09-06T20:31:43,690 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:31:43,746 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultipleOverwrite Thread=583, OpenFileDescriptor=660, MaxFileDescriptor=60000, SystemLoadAverage=229, ProcessCount=167, AvailableMemoryMB=3757 2023-09-06T20:31:43,746 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(130): Thread=583 is superior to 500 2023-09-06T20:31:43,750 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] backup.TestFullRestore(216): create full backup image on multiple tables 2023-09-06T20:31:43,752 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3165858b to 127.0.0.1:53515 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:31:43,761 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ea96ee5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:31:43,774 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:31:43,776 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:58630, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:31:43,811 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:31:43,813 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:35826, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:31:43,826 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'backup'} 2023-09-06T20:31:43,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-09-06T20:31:43,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=25 2023-09-06T20:31:43,845 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:31:43,851 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=25, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 23 msec 2023-09-06T20:31:43,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=25 2023-09-06T20:31:43,949 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:31:43,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-09-06T20:31:43,954 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:31:43,954 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:31:43,954 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2023-09-06T20:31:43,955 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:31:43,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-09-06T20:31:44,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-09-06T20:31:44,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-09-06T20:31:44,374 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => cf63a85c53279bc94811b50578f31281, NAME => 'backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:31:44,394 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:31:44,394 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing cf63a85c53279bc94811b50578f31281, disabling compactions & flushes 2023-09-06T20:31:44,394 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:31:44,394 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:31:44,394 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. after waiting 0 ms 2023-09-06T20:31:44,394 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:31:44,394 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:31:44,394 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for cf63a85c53279bc94811b50578f31281: 2023-09-06T20:31:44,396 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:31:44,397 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1694032304396"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032304396"}]},"ts":"1694032304396"} 2023-09-06T20:31:44,399 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:31:44,400 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:31:44,400 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032304400"}]},"ts":"1694032304400"} 2023-09-06T20:31:44,402 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-09-06T20:31:44,406 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=cf63a85c53279bc94811b50578f31281, ASSIGN}] 2023-09-06T20:31:44,409 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=cf63a85c53279bc94811b50578f31281, ASSIGN 2023-09-06T20:31:44,410 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=cf63a85c53279bc94811b50578f31281, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36961,1694032287907; forceNewPlan=false, retain=false 2023-09-06T20:31:44,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-09-06T20:31:44,561 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=cf63a85c53279bc94811b50578f31281, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:44,564 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; OpenRegionProcedure cf63a85c53279bc94811b50578f31281, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:31:44,721 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(130): Open backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:31:44,721 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7254): Opening region: {ENCODED => cf63a85c53279bc94811b50578f31281, NAME => 'backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:31:44,722 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:31:44,722 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system cf63a85c53279bc94811b50578f31281 2023-09-06T20:31:44,722 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(888): Instantiated backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:31:44,723 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7296): checking encryption for cf63a85c53279bc94811b50578f31281 2023-09-06T20:31:44,723 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7299): checking classloading for cf63a85c53279bc94811b50578f31281 2023-09-06T20:31:44,724 INFO [StoreOpener-cf63a85c53279bc94811b50578f31281-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region cf63a85c53279bc94811b50578f31281 2023-09-06T20:31:44,727 INFO [StoreOpener-cf63a85c53279bc94811b50578f31281-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf63a85c53279bc94811b50578f31281 columnFamilyName meta 2023-09-06T20:31:44,727 DEBUG [StoreOpener-cf63a85c53279bc94811b50578f31281-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:31:44,728 INFO [StoreOpener-cf63a85c53279bc94811b50578f31281-1 {}] regionserver.HStore(324): Store=cf63a85c53279bc94811b50578f31281/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:31:44,728 INFO [StoreOpener-cf63a85c53279bc94811b50578f31281-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region cf63a85c53279bc94811b50578f31281 2023-09-06T20:31:44,729 INFO [StoreOpener-cf63a85c53279bc94811b50578f31281-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf63a85c53279bc94811b50578f31281 columnFamilyName session 2023-09-06T20:31:44,729 DEBUG [StoreOpener-cf63a85c53279bc94811b50578f31281-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:31:44,730 INFO [StoreOpener-cf63a85c53279bc94811b50578f31281-1 {}] regionserver.HStore(324): Store=cf63a85c53279bc94811b50578f31281/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:31:44,731 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281 2023-09-06T20:31:44,731 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281 2023-09-06T20:31:44,733 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-09-06T20:31:44,735 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1076): writing seq id for cf63a85c53279bc94811b50578f31281 2023-09-06T20:31:44,737 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:31:44,738 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): Opened cf63a85c53279bc94811b50578f31281; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10045675680, jitterRate=-0.06442354619503021}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-09-06T20:31:44,738 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(992): Region open journal for cf63a85c53279bc94811b50578f31281: 2023-09-06T20:31:44,739 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281., pid=28, masterSystemTime=1694032304716 2023-09-06T20:31:44,741 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:31:44,742 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(158): Opened backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:31:44,742 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=cf63a85c53279bc94811b50578f31281, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:44,747 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=28, resume processing ppid=27 2023-09-06T20:31:44,747 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=28, ppid=27, state=SUCCESS; OpenRegionProcedure cf63a85c53279bc94811b50578f31281, server=jenkins-hbase3.apache.org,36961,1694032287907 in 180 msec 2023-09-06T20:31:44,749 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=27, resume processing ppid=26 2023-09-06T20:31:44,749 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=cf63a85c53279bc94811b50578f31281, ASSIGN in 341 msec 2023-09-06T20:31:44,750 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:31:44,750 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032304750"}]},"ts":"1694032304750"} 2023-09-06T20:31:44,752 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-09-06T20:31:44,755 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:31:44,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=26, state=SUCCESS; CreateTableProcedure table=backup:system in 806 msec 2023-09-06T20:31:45,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-09-06T20:31:45,064 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: backup:system, procId: 26 completed 2023-09-06T20:31:45,079 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:31:45,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-09-06T20:31:45,083 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:31:45,083 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:31:45,083 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2023-09-06T20:31:45,084 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:31:45,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-09-06T20:31:45,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-09-06T20:31:45,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-09-06T20:31:45,498 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 3a1cfaf5b9def8649351000383a2d6de, NAME => 'backup:system_bulk,,1694032305079.3a1cfaf5b9def8649351000383a2d6de.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:31:45,512 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1694032305079.3a1cfaf5b9def8649351000383a2d6de.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:31:45,512 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing 3a1cfaf5b9def8649351000383a2d6de, disabling compactions & flushes 2023-09-06T20:31:45,512 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1694032305079.3a1cfaf5b9def8649351000383a2d6de. 2023-09-06T20:31:45,512 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1694032305079.3a1cfaf5b9def8649351000383a2d6de. 2023-09-06T20:31:45,512 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1694032305079.3a1cfaf5b9def8649351000383a2d6de. after waiting 0 ms 2023-09-06T20:31:45,512 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1694032305079.3a1cfaf5b9def8649351000383a2d6de. 2023-09-06T20:31:45,512 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1694032305079.3a1cfaf5b9def8649351000383a2d6de. 2023-09-06T20:31:45,512 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for 3a1cfaf5b9def8649351000383a2d6de: 2023-09-06T20:31:45,514 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:31:45,514 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1694032305079.3a1cfaf5b9def8649351000383a2d6de.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1694032305514"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032305514"}]},"ts":"1694032305514"} 2023-09-06T20:31:45,516 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:31:45,517 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:31:45,518 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032305517"}]},"ts":"1694032305517"} 2023-09-06T20:31:45,520 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-09-06T20:31:45,525 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=3a1cfaf5b9def8649351000383a2d6de, ASSIGN}] 2023-09-06T20:31:45,528 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=3a1cfaf5b9def8649351000383a2d6de, ASSIGN 2023-09-06T20:31:45,529 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=3a1cfaf5b9def8649351000383a2d6de, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36961,1694032287907; forceNewPlan=false, retain=false 2023-09-06T20:31:45,680 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=3a1cfaf5b9def8649351000383a2d6de, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:45,683 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE; OpenRegionProcedure 3a1cfaf5b9def8649351000383a2d6de, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:31:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-09-06T20:31:45,747 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16940322964963 2023-09-06T20:31:45,747 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-16940322964963 Metrics about Tables on a single HBase RegionServer 2023-09-06T20:31:45,748 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-09-06T20:31:45,748 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-09-06T20:31:45,749 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16940322964962 2023-09-06T20:31:45,749 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-16940322964962 Metrics about Tables on a single HBase RegionServer 2023-09-06T20:31:45,750 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16940322964961 2023-09-06T20:31:45,750 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-16940322964961 Metrics about Tables on a single HBase RegionServer 2023-09-06T20:31:45,750 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1694032296496 2023-09-06T20:31:45,750 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1694032296496 Metrics about Tables on a single HBase RegionServer 2023-09-06T20:31:45,839 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1694032305079.3a1cfaf5b9def8649351000383a2d6de. 2023-09-06T20:31:45,840 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7254): Opening region: {ENCODED => 3a1cfaf5b9def8649351000383a2d6de, NAME => 'backup:system_bulk,,1694032305079.3a1cfaf5b9def8649351000383a2d6de.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:31:45,840 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:31:45,840 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk 3a1cfaf5b9def8649351000383a2d6de 2023-09-06T20:31:45,840 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1694032305079.3a1cfaf5b9def8649351000383a2d6de.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:31:45,840 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7296): checking encryption for 3a1cfaf5b9def8649351000383a2d6de 2023-09-06T20:31:45,840 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7299): checking classloading for 3a1cfaf5b9def8649351000383a2d6de 2023-09-06T20:31:45,842 INFO [StoreOpener-3a1cfaf5b9def8649351000383a2d6de-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 3a1cfaf5b9def8649351000383a2d6de 2023-09-06T20:31:45,844 INFO [StoreOpener-3a1cfaf5b9def8649351000383a2d6de-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3a1cfaf5b9def8649351000383a2d6de columnFamilyName meta 2023-09-06T20:31:45,844 DEBUG [StoreOpener-3a1cfaf5b9def8649351000383a2d6de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:31:45,844 INFO [StoreOpener-3a1cfaf5b9def8649351000383a2d6de-1 {}] regionserver.HStore(324): Store=3a1cfaf5b9def8649351000383a2d6de/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:31:45,844 INFO [StoreOpener-3a1cfaf5b9def8649351000383a2d6de-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 3a1cfaf5b9def8649351000383a2d6de 2023-09-06T20:31:45,846 INFO [StoreOpener-3a1cfaf5b9def8649351000383a2d6de-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3a1cfaf5b9def8649351000383a2d6de columnFamilyName session 2023-09-06T20:31:45,846 DEBUG [StoreOpener-3a1cfaf5b9def8649351000383a2d6de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:31:45,847 INFO [StoreOpener-3a1cfaf5b9def8649351000383a2d6de-1 {}] regionserver.HStore(324): Store=3a1cfaf5b9def8649351000383a2d6de/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:31:45,848 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system_bulk/3a1cfaf5b9def8649351000383a2d6de 2023-09-06T20:31:45,848 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system_bulk/3a1cfaf5b9def8649351000383a2d6de 2023-09-06T20:31:45,850 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-09-06T20:31:45,851 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1076): writing seq id for 3a1cfaf5b9def8649351000383a2d6de 2023-09-06T20:31:45,854 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system_bulk/3a1cfaf5b9def8649351000383a2d6de/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:31:45,854 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): Opened 3a1cfaf5b9def8649351000383a2d6de; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10515284960, jitterRate=-0.020687773823738098}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-09-06T20:31:45,855 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(992): Region open journal for 3a1cfaf5b9def8649351000383a2d6de: 2023-09-06T20:31:45,856 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system_bulk,,1694032305079.3a1cfaf5b9def8649351000383a2d6de., pid=31, masterSystemTime=1694032305835 2023-09-06T20:31:45,858 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system_bulk,,1694032305079.3a1cfaf5b9def8649351000383a2d6de. 2023-09-06T20:31:45,858 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1694032305079.3a1cfaf5b9def8649351000383a2d6de. 2023-09-06T20:31:45,859 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=3a1cfaf5b9def8649351000383a2d6de, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:45,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=31, resume processing ppid=30 2023-09-06T20:31:45,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=31, ppid=30, state=SUCCESS; OpenRegionProcedure 3a1cfaf5b9def8649351000383a2d6de, server=jenkins-hbase3.apache.org,36961,1694032287907 in 178 msec 2023-09-06T20:31:45,865 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=30, resume processing ppid=29 2023-09-06T20:31:45,865 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=30, ppid=29, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=3a1cfaf5b9def8649351000383a2d6de, ASSIGN in 338 msec 2023-09-06T20:31:45,866 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:31:45,867 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032305866"}]},"ts":"1694032305866"} 2023-09-06T20:31:45,868 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-09-06T20:31:45,871 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:31:45,873 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=29, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 792 msec 2023-09-06T20:31:46,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-09-06T20:31:46,191 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: backup:system_bulk, procId: 29 completed 2023-09-06T20:31:46,203 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:31:46,250 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:31:46,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032306250 (current time:1694032306250). 2023-09-06T20:31:46,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:31:46,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:31:46,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x65b03253 to 127.0.0.1:53515 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:31:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@781df40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:31:46,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:31:46,263 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:58638, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:31:46,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x65b03253 to 127.0.0.1:53515 2023-09-06T20:31:46,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:31:46,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:31:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:31:46,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-09-06T20:31:46,280 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:31:46,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-09-06T20:31:46,285 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:31:46,288 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:31:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-09-06T20:31:46,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-09-06T20:31:46,699 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:31:46,702 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281}] 2023-09-06T20:31:46,710 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 2023-09-06T20:31:46,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2023-09-06T20:31:46,871 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:31:46,874 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2819): Flushing cf63a85c53279bc94811b50578f31281 2/2 column families, dataSize=45 B heapSize=632 B 2023-09-06T20:31:46,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-09-06T20:31:46,951 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/4ced064cdbd34bd6b594c76f049fa4ee is 49, key is activesession:/session:c/1694032306235/Put/seqid=0 2023-09-06T20:31:47,370 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/4ced064cdbd34bd6b594c76f049fa4ee 2023-09-06T20:31:47,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-09-06T20:31:47,460 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/4ced064cdbd34bd6b594c76f049fa4ee as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/4ced064cdbd34bd6b594c76f049fa4ee 2023-09-06T20:31:47,471 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/4ced064cdbd34bd6b594c76f049fa4ee, entries=1, sequenceid=5, filesize=4.9 K 2023-09-06T20:31:47,475 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for cf63a85c53279bc94811b50578f31281 in 601ms, sequenceid=5, compaction requested=false 2023-09-06T20:31:47,475 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-09-06T20:31:47,478 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2520): Flush status journal for cf63a85c53279bc94811b50578f31281: 2023-09-06T20:31:47,478 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. for snapshot_backup_system completed. 2023-09-06T20:31:47,479 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:31:47,485 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:31:47,490 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-09-06T20:31:47,490 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/4ced064cdbd34bd6b594c76f049fa4ee] hfiles 2023-09-06T20:31:47,490 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/4ced064cdbd34bd6b594c76f049fa4ee for snapshot=snapshot_backup_system 2023-09-06T20:31:47,906 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:31:47,910 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=33 2023-09-06T20:31:47,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=33 2023-09-06T20:31:47,913 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region cf63a85c53279bc94811b50578f31281 2023-09-06T20:31:47,915 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 2023-09-06T20:31:47,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=33, resume processing ppid=32 2023-09-06T20:31:47,918 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:31:47,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=33, ppid=32, state=SUCCESS; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 in 1.2140 sec 2023-09-06T20:31:47,920 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:31:47,923 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:31:47,924 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:31:47,926 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:31:47,951 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:31:47,966 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:31:47,966 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:31:47,969 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:31:47,969 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-09-06T20:31:47,971 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=32, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.6960 sec 2023-09-06T20:31:48,239 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-09-06T20:31:48,240 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-16940322964963' 2023-09-06T20:31:48,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-09-06T20:31:48,389 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: backup:system, procId: 32 completed 2023-09-06T20:31:48,389 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(115): Backup backup_1694032303765 started at 1694032308389. 2023-09-06T20:31:48,419 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(119): Backup session backup_1694032303765 has been started. 2023-09-06T20:31:48,422 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-09-06T20:31:48,436 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-09-06T20:31:48,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-09-06T20:31:48,444 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-09-06T20:31:48,444 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-09-06T20:31:48,445 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-09-06T20:31:48,445 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-09-06T20:31:48,446 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-09-06T20:31:48,446 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:31:48,447 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-09-06T20:31:48,447 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:48,447 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:31:48,448 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:31:48,448 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:48,448 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-09-06T20:31:48,448 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-09-06T20:31:48,448 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-09-06T20:31:48,449 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-09-06T20:31:48,449 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:31:48,449 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-09-06T20:31:48,451 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-09-06T20:31:48,451 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-09-06T20:31:48,451 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-09-06T20:31:48,451 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-09-06T20:31:48,452 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-09-06T20:31:48,452 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-09-06T20:31:48,452 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,36961,1694032287907' joining acquired barrier for procedure (rolllog) in zk 2023-09-06T20:31:48,455 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:48,455 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-09-06T20:31:48,455 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:48,455 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:31:48,455 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:31:48,455 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-09-06T20:31:48,455 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-09-06T20:31:48,455 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:31:48,456 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:31:48,456 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:31:48,457 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:48,457 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:31:48,457 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,36961,1694032287907' joining acquired barrier for procedure 'rolllog' on coordinator 2023-09-06T20:31:48,458 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-09-06T20:31:48,458 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@6abcdb89[Count = 0] remaining members to acquire global barrier 2023-09-06T20:31:48,458 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-09-06T20:31:48,460 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-09-06T20:31:48,460 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-09-06T20:31:48,460 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-09-06T20:31:48,460 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-09-06T20:31:48,460 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:48,460 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-09-06T20:31:48,461 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-09-06T20:31:48,461 DEBUG [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:48,462 INFO [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1694032290035 highest: 1694032290035 on jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:48,462 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta:.meta(num 1694032290534) roll requested 2023-09-06T20:31:48,473 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032308462.meta, exclude list is [], retry=0 2023-09-06T20:31:48,481 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK] 2023-09-06T20:31:48,485 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032290534.meta with entries=36, filesize=11.13 KB; new WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032308462.meta 2023-09-06T20:31:48,487 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK]] 2023-09-06T20:31:48,487 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032290534.meta is not closed yet, will try archiving it next time 2023-09-06T20:31:48,487 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36961%2C1694032287907:(num 1694032290035) roll requested 2023-09-06T20:31:48,503 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032308487, exclude list is [], retry=0 2023-09-06T20:31:48,507 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK] 2023-09-06T20:31:48,511 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032290035 with entries=19, filesize=4.02 KB; new WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032308487 2023-09-06T20:31:48,511 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK]] 2023-09-06T20:31:48,511 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032290035 is not closed yet, will try archiving it next time 2023-09-06T20:31:48,522 DEBUG [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2023-09-06T20:31:48,522 INFO [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1694032308487 on jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:48,523 DEBUG [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:31:48,525 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:35832, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-09-06T20:31:48,545 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-09-06T20:31:48,545 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,36961,1694032287907' in zk 2023-09-06T20:31:48,549 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:48,549 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-09-06T20:31:48,549 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:31:48,549 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:48,549 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-09-06T20:31:48,549 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:31:48,549 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:31:48,550 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:31:48,551 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:31:48,551 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:31:48,551 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:48,552 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:31:48,552 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:31:48,552 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:48,553 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,36961,1694032287907': 2023-09-06T20:31:48,553 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-09-06T20:31:48,553 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,36961,1694032287907' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-09-06T20:31:48,553 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-09-06T20:31:48,553 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-09-06T20:31:48,553 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-09-06T20:31:48,554 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-09-06T20:31:48,556 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-09-06T20:31:48,556 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-09-06T20:31:48,557 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-09-06T20:31:48,557 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-09-06T20:31:48,557 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-09-06T20:31:48,557 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:31:48,557 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:31:48,557 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:31:48,557 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:31:48,557 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:31:48,557 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:31:48,557 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:48,558 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:31:48,558 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:31:48,558 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:31:48,558 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:31:48,559 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:48,559 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:48,559 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:31:48,560 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:31:48,560 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:48,567 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:48,568 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-09-06T20:31:48,568 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-09-06T20:31:48,568 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-09-06T20:31:48,568 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-09-06T20:31:48,568 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:31:48,568 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:48,568 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:31:48,568 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-09-06T20:31:48,568 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-09-06T20:31:48,568 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-09-06T20:31:48,568 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:31:48,569 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-09-06T20:31:48,569 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-09-06T20:31:48,569 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:31:48,569 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:31:48,571 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin(2838): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-09-06T20:31:48,571 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin(2846): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-09-06T20:31:48,671 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin(2852): Getting current status of procedure from master... 2023-09-06T20:31:48,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-09-06T20:31:48,682 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1694032308681_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } 2023-09-06T20:31:48,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032308682 (current time:1694032308682). 2023-09-06T20:31:48,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:31:48,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1694032308681_ns2_test-16940322964961 VERSION not specified, setting to 2 2023-09-06T20:31:48,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2ed7bf3d to 127.0.0.1:53515 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:31:48,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1339dd07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:31:48,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:31:48,692 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:58642, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:31:48,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2ed7bf3d to 127.0.0.1:53515 2023-09-06T20:31:48,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:31:48,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:31:48,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032308681_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } 2023-09-06T20:31:48,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1694032308681_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-09-06T20:31:48,697 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032308681_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:31:48,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:31:48,698 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032308681_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:31:48,700 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032308681_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:31:48,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:31:48,860 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:31:49,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:31:49,110 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032308681_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:31:49,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 1932330ab1e654ad2ac9ae28fd5fcdb3}] 2023-09-06T20:31:49,113 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 1932330ab1e654ad2ac9ae28fd5fcdb3 2023-09-06T20:31:49,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2023-09-06T20:31:49,265 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16940322964961,,1694032299223.1932330ab1e654ad2ac9ae28fd5fcdb3. 2023-09-06T20:31:49,267 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2819): Flushing 1932330ab1e654ad2ac9ae28fd5fcdb3 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-09-06T20:31:49,286 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/1932330ab1e654ad2ac9ae28fd5fcdb3/.tmp/f/c196aaa261e74a059e3eb2c060e5b44a is 37, key is row10/f:q1/1694032300351/Put/seqid=0 2023-09-06T20:31:49,296 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/1932330ab1e654ad2ac9ae28fd5fcdb3/.tmp/f/c196aaa261e74a059e3eb2c060e5b44a 2023-09-06T20:31:49,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:31:49,307 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/1932330ab1e654ad2ac9ae28fd5fcdb3/.tmp/f/c196aaa261e74a059e3eb2c060e5b44a as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/1932330ab1e654ad2ac9ae28fd5fcdb3/f/c196aaa261e74a059e3eb2c060e5b44a 2023-09-06T20:31:49,317 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/1932330ab1e654ad2ac9ae28fd5fcdb3/f/c196aaa261e74a059e3eb2c060e5b44a, entries=99, sequenceid=103, filesize=8.2 K 2023-09-06T20:31:49,319 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 1932330ab1e654ad2ac9ae28fd5fcdb3 in 53ms, sequenceid=103, compaction requested=false 2023-09-06T20:31:49,319 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2520): Flush status journal for 1932330ab1e654ad2ac9ae28fd5fcdb3: 2023-09-06T20:31:49,319 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16940322964961,,1694032299223.1932330ab1e654ad2ac9ae28fd5fcdb3. for snapshot_1694032308681_ns2_test-16940322964961 completed. 2023-09-06T20:31:49,320 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16940322964961,,1694032299223.1932330ab1e654ad2ac9ae28fd5fcdb3.' region-info for snapshot=snapshot_1694032308681_ns2_test-16940322964961 2023-09-06T20:31:49,320 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:31:49,320 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/1932330ab1e654ad2ac9ae28fd5fcdb3/f/c196aaa261e74a059e3eb2c060e5b44a] hfiles 2023-09-06T20:31:49,320 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/1932330ab1e654ad2ac9ae28fd5fcdb3/f/c196aaa261e74a059e3eb2c060e5b44a for snapshot=snapshot_1694032308681_ns2_test-16940322964961 2023-09-06T20:31:49,731 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16940322964961,,1694032299223.1932330ab1e654ad2ac9ae28fd5fcdb3. 2023-09-06T20:31:49,732 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=35 2023-09-06T20:31:49,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=35 2023-09-06T20:31:49,732 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1694032308681_ns2_test-16940322964961 on region 1932330ab1e654ad2ac9ae28fd5fcdb3 2023-09-06T20:31:49,734 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 1932330ab1e654ad2ac9ae28fd5fcdb3 2023-09-06T20:31:49,737 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=35, resume processing ppid=34 2023-09-06T20:31:49,737 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=35, ppid=34, state=SUCCESS; SnapshotRegionProcedure 1932330ab1e654ad2ac9ae28fd5fcdb3 in 624 msec 2023-09-06T20:31:49,737 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032308681_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:31:49,738 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032308681_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:31:49,738 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032308681_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:31:49,738 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1694032308681_ns2_test-16940322964961 2023-09-06T20:31:49,739 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032308681_ns2_test-16940322964961 2023-09-06T20:31:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:31:50,159 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032308681_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:31:50,167 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032308681_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:31:50,168 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032308681_ns2_test-16940322964961 to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032308681_ns2_test-16940322964961 2023-09-06T20:31:50,169 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032308681_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:31:50,169 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1694032308681_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-09-06T20:31:50,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=34, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1694032308681_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } in 1.4750 sec 2023-09-06T20:31:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:31:50,807 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: ns2:test-16940322964961, procId: 34 completed 2023-09-06T20:31:50,808 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1694032310807_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } 2023-09-06T20:31:50,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032310808 (current time:1694032310808). 2023-09-06T20:31:50,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:31:50,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1694032310807_ns3_test-16940322964962 VERSION not specified, setting to 2 2023-09-06T20:31:50,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x695c6c4b to 127.0.0.1:53515 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:31:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ae1080a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:31:50,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:31:50,818 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:47256, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:31:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x695c6c4b to 127.0.0.1:53515 2023-09-06T20:31:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:31:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:31:50,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=36, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1694032310807_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } 2023-09-06T20:31:50,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1694032310807_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-09-06T20:31:50,823 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1694032310807_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:31:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-09-06T20:31:50,824 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1694032310807_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:31:50,826 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1694032310807_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:31:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-09-06T20:31:51,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-09-06T20:31:51,240 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1694032310807_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:31:51,240 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 18ddf30d0a5d8ca482131f8c37827552}] 2023-09-06T20:31:51,243 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 18ddf30d0a5d8ca482131f8c37827552 2023-09-06T20:31:51,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=37 2023-09-06T20:31:51,395 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16940322964962,,1694032300440.18ddf30d0a5d8ca482131f8c37827552. 2023-09-06T20:31:51,397 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2520): Flush status journal for 18ddf30d0a5d8ca482131f8c37827552: 2023-09-06T20:31:51,397 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16940322964962,,1694032300440.18ddf30d0a5d8ca482131f8c37827552. for snapshot_1694032310807_ns3_test-16940322964962 completed. 2023-09-06T20:31:51,397 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16940322964962,,1694032300440.18ddf30d0a5d8ca482131f8c37827552.' region-info for snapshot=snapshot_1694032310807_ns3_test-16940322964962 2023-09-06T20:31:51,397 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:31:51,397 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-09-06T20:31:51,406 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16940322964962,,1694032300440.18ddf30d0a5d8ca482131f8c37827552. 2023-09-06T20:31:51,406 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=37 2023-09-06T20:31:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=37 2023-09-06T20:31:51,407 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1694032310807_ns3_test-16940322964962 on region 18ddf30d0a5d8ca482131f8c37827552 2023-09-06T20:31:51,407 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 18ddf30d0a5d8ca482131f8c37827552 2023-09-06T20:31:51,410 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=37, resume processing ppid=36 2023-09-06T20:31:51,410 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=37, ppid=36, state=SUCCESS; SnapshotRegionProcedure 18ddf30d0a5d8ca482131f8c37827552 in 168 msec 2023-09-06T20:31:51,410 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1694032310807_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:31:51,411 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1694032310807_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:31:51,412 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1694032310807_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:31:51,412 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1694032310807_ns3_test-16940322964962 2023-09-06T20:31:51,413 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032310807_ns3_test-16940322964962 2023-09-06T20:31:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-09-06T20:31:51,828 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1694032310807_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:31:51,834 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1694032310807_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:31:51,835 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032310807_ns3_test-16940322964962 to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032310807_ns3_test-16940322964962 2023-09-06T20:31:51,836 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1694032310807_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:31:51,837 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1694032310807_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-09-06T20:31:51,838 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=36, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1694032310807_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } in 1.0170 sec 2023-09-06T20:31:51,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-09-06T20:31:51,930 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: ns3:test-16940322964962, procId: 36 completed 2023-09-06T20:31:51,930 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1694032303765 2023-09-06T20:31:51,930 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-09-06T20:31:51,932 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-09-06T20:31:51,934 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1694032303765_ns3:test-16940322964962 2023-09-06T20:31:51,934 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1694032310807_ns3_test-16940322964962 to hdfs://localhost:39327/backupUT/backup_1694032303765/ns3/test-16940322964962/ 2023-09-06T20:31:51,944 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-09-06T20:31:51,981 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:31:51,981 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37419,1694032284546_1558825974_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:39327/backupUT/backup_1694032303765/ns3/test-16940322964962, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:39327/backupUT/backup_1694032303765/ns3/test-16940322964962/.hbase-snapshot/.tmp/snapshot_1694032310807_ns3_test-16940322964962 2023-09-06T20:31:51,982 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1, snapshotDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032310807_ns3_test-16940322964962. 2023-09-06T20:31:51,989 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032310807_ns3_test-16940322964962 to hdfs://localhost:39327/backupUT/backup_1694032303765/ns3/test-16940322964962/.hbase-snapshot/.tmp/snapshot_1694032310807_ns3_test-16940322964962 2023-09-06T20:31:52,037 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:31:52,038 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:31:52,038 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:31:52,039 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:31:53,052 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop-7569057211174868303.jar 2023-09-06T20:31:53,053 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:31:53,053 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:31:53,054 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:31:53,055 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:31:53,055 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:31:53,056 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:31:53,056 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:31:53,057 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:31:53,058 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:31:53,059 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:31:53,060 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:31:53,061 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:31:53,061 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:31:53,062 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:31:53,063 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:31:53,063 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:31:53,063 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:31:53,064 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:31:53,064 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:31:53,065 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:31:53,065 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:31:53,070 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:31:53,070 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:31:53,071 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:31:53,071 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:31:53,071 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:31:53,072 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:31:53,073 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:31:53,169 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-09-06T20:31:53,200 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1694032310807_ns3_test-16940322964962' hfile list 2023-09-06T20:31:53,445 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032313389/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:31:53,445 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032313389/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:31:54,553 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-09-06T20:31:54,554 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-09-06T20:31:54,565 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1694032310807_ns3_test-16940322964962 2023-09-06T20:31:54,565 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1694032310807_ns3_test-16940322964962 finished. 2023-09-06T20:31:54,565 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1694032303765_ns2:test-16940322964961 2023-09-06T20:31:54,565 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1694032308681_ns2_test-16940322964961 to hdfs://localhost:39327/backupUT/backup_1694032303765/ns2/test-16940322964961/ 2023-09-06T20:31:54,565 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-09-06T20:31:54,606 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:31:54,606 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37419,1694032284546_-968130589_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:39327/backupUT/backup_1694032303765/ns2/test-16940322964961, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:39327/backupUT/backup_1694032303765/ns2/test-16940322964961/.hbase-snapshot/.tmp/snapshot_1694032308681_ns2_test-16940322964961 2023-09-06T20:31:54,606 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1, snapshotDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032308681_ns2_test-16940322964961. 2023-09-06T20:31:54,618 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032308681_ns2_test-16940322964961 to hdfs://localhost:39327/backupUT/backup_1694032303765/ns2/test-16940322964961/.hbase-snapshot/.tmp/snapshot_1694032308681_ns2_test-16940322964961 2023-09-06T20:31:55,049 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:31:55,049 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:31:55,050 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:31:55,050 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:31:55,747 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-09-06T20:31:55,747 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-09-06T20:31:56,130 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop-5325691400329680973.jar 2023-09-06T20:31:56,131 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:31:56,131 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:31:56,132 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:31:56,132 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:31:56,133 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:31:56,133 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:31:56,134 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:31:56,134 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:31:56,134 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:31:56,135 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:31:56,136 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:31:56,136 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:31:56,137 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:31:56,137 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:31:56,137 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:31:56,138 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:31:56,138 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:31:56,139 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:31:56,139 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:31:56,140 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:31:56,140 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:31:56,141 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:31:56,141 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:31:56,142 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:31:56,142 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:31:56,143 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:31:56,143 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:31:56,144 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:31:56,171 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-09-06T20:31:56,190 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1694032308681_ns2_test-16940322964961' hfile list 2023-09-06T20:31:56,199 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-09-06T20:31:56,315 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032316268/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:31:56,315 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032316268/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:31:56,444 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-09-06T20:31:56,478 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/1932330ab1e654ad2ac9ae28fd5fcdb3/f/c196aaa261e74a059e3eb2c060e5b44a output=hdfs://localhost:39327/backupUT/backup_1694032303765/ns2/test-16940322964961/archive/data/ns2/test-16940322964961/1932330ab1e654ad2ac9ae28fd5fcdb3/f/c196aaa261e74a059e3eb2c060e5b44a 2023-09-06T20:31:56,478 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-09-06T20:31:57,364 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-09-06T20:31:57,366 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-09-06T20:31:57,374 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1694032308681_ns2_test-16940322964961 2023-09-06T20:31:57,374 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1694032308681_ns2_test-16940322964961 finished. 2023-09-06T20:31:57,374 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1014): ns3:test-16940322964962 2023-09-06T20:31:57,374 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1014): ns2:test-16940322964961 2023-09-06T20:31:57,448 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1694032303765 2023-09-06T20:31:57,448 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-09-06T20:31:57,861 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:39327/backupUT/backup_1694032303765/.backup.manifest 2023-09-06T20:31:57,861 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1694032303765 2023-09-06T20:31:57,861 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-09-06T20:31:58,269 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:39327/backupUT/backup_1694032303765/.backup.manifest 2023-09-06T20:31:58,269 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(358): Backup backup_1694032303765 finished: type=FULL,tablelist=ns3:test-16940322964962;ns2:test-16940322964961,targetRootDir=hdfs://localhost:39327/backupUT,startts=1694032308389,completets=1694032317444,bytescopied=0 2023-09-06T20:31:58,269 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-09-06T20:31:58,269 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1694032310807_ns3_test-16940322964962 2023-09-06T20:31:58,279 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1694032310807_ns3_test-16940322964962" 2023-09-06T20:31:58,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1694032310807_ns3_test-16940322964962 2023-09-06T20:31:58,285 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1694032310807_ns3_test-16940322964962 for backup backup_1694032303765 succeeded. 2023-09-06T20:31:58,285 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1694032308681_ns2_test-16940322964961 2023-09-06T20:31:58,286 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1694032308681_ns2_test-16940322964961" 2023-09-06T20:31:58,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1694032308681_ns2_test-16940322964961 2023-09-06T20:31:58,290 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1694032308681_ns2_test-16940322964961 for backup backup_1694032303765 succeeded. 2023-09-06T20:31:58,291 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:31:58,304 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:31:58,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:31:58,307 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:31:58,314 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:31:58,337 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(376): Backup backup_1694032303765 completed. 2023-09-06T20:31:58,337 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:31:58,337 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3165858b to 127.0.0.1:53515 2023-09-06T20:31:58,337 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:31:58,368 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:39327/backupUT/backup_1694032303765/.backup.manifest 2023-09-06T20:31:58,371 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032303765/.backup.manifest 2023-09-06T20:31:58,372 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:39327/backupUT/backup_1694032303765/.backup.manifest 2023-09-06T20:31:58,374 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032303765/.backup.manifest 2023-09-06T20:31:58,381 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:39327/backupUT/backup_1694032303765/.backup.manifest 2023-09-06T20:31:58,383 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032303765/.backup.manifest 2023-09-06T20:31:58,383 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(148): Restoring 'ns2:test-16940322964961' to 'ns2:test-16940322964961' from full backup image hdfs://localhost:39327/backupUT/backup_1694032303765/ns2/test-16940322964961 2023-09-06T20:31:58,394 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(476): Truncating exising target table 'ns2:test-16940322964961', preserving region splits 2023-09-06T20:31:58,396 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin$18(967): Started disable of ns2:test-16940322964961 2023-09-06T20:31:58,403 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable ns2:test-16940322964961 2023-09-06T20:31:58,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=38, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns2:test-16940322964961 2023-09-06T20:31:58,418 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16940322964961","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032318418"}]},"ts":"1694032318418"} 2023-09-06T20:31:58,420 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16940322964961, state=DISABLING in hbase:meta 2023-09-06T20:31:58,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-09-06T20:31:58,422 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set ns2:test-16940322964961 to state=DISABLING 2023-09-06T20:31:58,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:test-16940322964961, region=1932330ab1e654ad2ac9ae28fd5fcdb3, UNASSIGN}] 2023-09-06T20:31:58,425 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:test-16940322964961, region=1932330ab1e654ad2ac9ae28fd5fcdb3, UNASSIGN 2023-09-06T20:31:58,426 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=39 updating hbase:meta row=1932330ab1e654ad2ac9ae28fd5fcdb3, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:58,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE; CloseRegionProcedure 1932330ab1e654ad2ac9ae28fd5fcdb3, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:31:58,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-09-06T20:31:58,591 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(119): Close 1932330ab1e654ad2ac9ae28fd5fcdb3 2023-09-06T20:31:58,591 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1668): Closing 1932330ab1e654ad2ac9ae28fd5fcdb3, disabling compactions & flushes 2023-09-06T20:31:58,591 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1690): Closing region ns2:test-16940322964961,,1694032299223.1932330ab1e654ad2ac9ae28fd5fcdb3. 2023-09-06T20:31:58,591 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16940322964961,,1694032299223.1932330ab1e654ad2ac9ae28fd5fcdb3. 2023-09-06T20:31:58,591 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16940322964961,,1694032299223.1932330ab1e654ad2ac9ae28fd5fcdb3. after waiting 0 ms 2023-09-06T20:31:58,591 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16940322964961,,1694032299223.1932330ab1e654ad2ac9ae28fd5fcdb3. 2023-09-06T20:31:58,599 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/1932330ab1e654ad2ac9ae28fd5fcdb3/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-09-06T20:31:58,600 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:31:58,601 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1909): Closed ns2:test-16940322964961,,1694032299223.1932330ab1e654ad2ac9ae28fd5fcdb3. 2023-09-06T20:31:58,601 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1622): Region close journal for 1932330ab1e654ad2ac9ae28fd5fcdb3: 2023-09-06T20:31:58,603 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(163): Closed 1932330ab1e654ad2ac9ae28fd5fcdb3 2023-09-06T20:31:58,604 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=39 updating hbase:meta row=1932330ab1e654ad2ac9ae28fd5fcdb3, regionState=CLOSED 2023-09-06T20:31:58,608 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=40, resume processing ppid=39 2023-09-06T20:31:58,608 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=40, ppid=39, state=SUCCESS; CloseRegionProcedure 1932330ab1e654ad2ac9ae28fd5fcdb3, server=jenkins-hbase3.apache.org,36961,1694032287907 in 178 msec 2023-09-06T20:31:58,611 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=39, resume processing ppid=38 2023-09-06T20:31:58,611 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=39, ppid=38, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16940322964961, region=1932330ab1e654ad2ac9ae28fd5fcdb3, UNASSIGN in 185 msec 2023-09-06T20:31:58,612 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16940322964961","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032318611"}]},"ts":"1694032318611"} 2023-09-06T20:31:58,613 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16940322964961, state=DISABLED in hbase:meta 2023-09-06T20:31:58,615 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set ns2:test-16940322964961 to state=DISABLED 2023-09-06T20:31:58,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=38, state=SUCCESS; DisableTableProcedure table=ns2:test-16940322964961 in 210 msec 2023-09-06T20:31:58,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-09-06T20:31:58,724 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin$TableFuture(3677): Operation: DISABLE, Table Name: ns2:test-16940322964961, procId: 38 completed 2023-09-06T20:31:58,725 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin$16(847): Started truncating ns2:test-16940322964961 2023-09-06T20:31:58,731 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$6(2480): Client=jenkins//172.31.12.81 truncate ns2:test-16940322964961 2023-09-06T20:31:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=41, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=ns2:test-16940322964961 preserveSplits=true) 2023-09-06T20:31:58,742 DEBUG [PEWorker-2 {}] procedure.TruncateTableProcedure(87): waiting for 'ns2:test-16940322964961' regions in transition 2023-09-06T20:31:58,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=41 2023-09-06T20:31:58,750 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/1932330ab1e654ad2ac9ae28fd5fcdb3 2023-09-06T20:31:58,755 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/1932330ab1e654ad2ac9ae28fd5fcdb3/f, FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/1932330ab1e654ad2ac9ae28fd5fcdb3/recovered.edits] 2023-09-06T20:31:58,763 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/1932330ab1e654ad2ac9ae28fd5fcdb3/f/c196aaa261e74a059e3eb2c060e5b44a to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/archive/data/ns2/test-16940322964961/1932330ab1e654ad2ac9ae28fd5fcdb3/f/c196aaa261e74a059e3eb2c060e5b44a 2023-09-06T20:31:58,767 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/1932330ab1e654ad2ac9ae28fd5fcdb3/recovered.edits/106.seqid to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/archive/data/ns2/test-16940322964961/1932330ab1e654ad2ac9ae28fd5fcdb3/recovered.edits/106.seqid 2023-09-06T20:31:58,768 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/1932330ab1e654ad2ac9ae28fd5fcdb3 2023-09-06T20:31:58,768 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(293): Archived ns2:test-16940322964961 regions 2023-09-06T20:31:58,782 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns2:test-16940322964961 from hbase:meta 2023-09-06T20:31:58,784 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(387): Removing 'ns2:test-16940322964961' descriptor. 2023-09-06T20:31:58,785 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(377): Removing 'ns2:test-16940322964961' from region states. 2023-09-06T20:31:58,786 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:test-16940322964961,,1694032299223.1932330ab1e654ad2ac9ae28fd5fcdb3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1694032318785"}]},"ts":"9223372036854775807"} 2023-09-06T20:31:58,788 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-06T20:31:58,788 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 1932330ab1e654ad2ac9ae28fd5fcdb3, NAME => 'ns2:test-16940322964961,,1694032299223.1932330ab1e654ad2ac9ae28fd5fcdb3.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:31:58,788 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(381): Marking 'ns2:test-16940322964961' as deleted. 2023-09-06T20:31:58,788 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:test-16940322964961","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1694032318788"}]},"ts":"9223372036854775807"} 2023-09-06T20:31:58,790 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1678): Deleted table ns2:test-16940322964961 state from META 2023-09-06T20:31:58,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=41 2023-09-06T20:31:59,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=41 2023-09-06T20:31:59,202 INFO [RegionOpenAndInit-ns2:test-16940322964961-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => cd8331e491339ba2021603869fc044dc, NAME => 'ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16940322964961', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:31:59,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=41 2023-09-06T20:31:59,609 DEBUG [RegionOpenAndInit-ns2:test-16940322964961-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:31:59,609 DEBUG [RegionOpenAndInit-ns2:test-16940322964961-pool-0 {}] regionserver.HRegion(1668): Closing cd8331e491339ba2021603869fc044dc, disabling compactions & flushes 2023-09-06T20:31:59,609 INFO [RegionOpenAndInit-ns2:test-16940322964961-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc. 2023-09-06T20:31:59,609 DEBUG [RegionOpenAndInit-ns2:test-16940322964961-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc. 2023-09-06T20:31:59,609 DEBUG [RegionOpenAndInit-ns2:test-16940322964961-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc. after waiting 0 ms 2023-09-06T20:31:59,609 DEBUG [RegionOpenAndInit-ns2:test-16940322964961-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc. 2023-09-06T20:31:59,609 INFO [RegionOpenAndInit-ns2:test-16940322964961-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc. 2023-09-06T20:31:59,609 DEBUG [RegionOpenAndInit-ns2:test-16940322964961-pool-0 {}] regionserver.HRegion(1622): Region close journal for cd8331e491339ba2021603869fc044dc: 2023-09-06T20:31:59,611 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1694032319610"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032319610"}]},"ts":"1694032319610"} 2023-09-06T20:31:59,612 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:31:59,613 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16940322964961","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032319613"}]},"ts":"1694032319613"} 2023-09-06T20:31:59,615 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16940322964961, state=ENABLING in hbase:meta 2023-09-06T20:31:59,620 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16940322964961, region=cd8331e491339ba2021603869fc044dc, ASSIGN}] 2023-09-06T20:31:59,621 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=42, ppid=41, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16940322964961, region=cd8331e491339ba2021603869fc044dc, ASSIGN 2023-09-06T20:31:59,622 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=42, ppid=41, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16940322964961, region=cd8331e491339ba2021603869fc044dc, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36961,1694032287907; forceNewPlan=false, retain=false 2023-09-06T20:31:59,774 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=42 updating hbase:meta row=cd8331e491339ba2021603869fc044dc, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:59,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE; OpenRegionProcedure cd8331e491339ba2021603869fc044dc, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:31:59,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=41 2023-09-06T20:31:59,933 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(130): Open ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc. 2023-09-06T20:31:59,933 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7254): Opening region: {ENCODED => cd8331e491339ba2021603869fc044dc, NAME => 'ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:31:59,933 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:31:59,933 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16940322964961 cd8331e491339ba2021603869fc044dc 2023-09-06T20:31:59,934 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(888): Instantiated ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:31:59,934 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7296): checking encryption for cd8331e491339ba2021603869fc044dc 2023-09-06T20:31:59,934 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7299): checking classloading for cd8331e491339ba2021603869fc044dc 2023-09-06T20:31:59,935 INFO [StoreOpener-cd8331e491339ba2021603869fc044dc-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region cd8331e491339ba2021603869fc044dc 2023-09-06T20:31:59,937 INFO [StoreOpener-cd8331e491339ba2021603869fc044dc-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cd8331e491339ba2021603869fc044dc columnFamilyName f 2023-09-06T20:31:59,937 DEBUG [StoreOpener-cd8331e491339ba2021603869fc044dc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:31:59,937 INFO [StoreOpener-cd8331e491339ba2021603869fc044dc-1 {}] regionserver.HStore(324): Store=cd8331e491339ba2021603869fc044dc/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:31:59,938 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc 2023-09-06T20:31:59,939 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc 2023-09-06T20:31:59,941 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1076): writing seq id for cd8331e491339ba2021603869fc044dc 2023-09-06T20:31:59,943 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:31:59,944 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): Opened cd8331e491339ba2021603869fc044dc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10649789280, jitterRate=-0.008161082863807678}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:31:59,944 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(992): Region open journal for cd8331e491339ba2021603869fc044dc: 2023-09-06T20:31:59,945 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc., pid=43, masterSystemTime=1694032319928 2023-09-06T20:31:59,947 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc. 2023-09-06T20:31:59,947 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(158): Opened ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc. 2023-09-06T20:31:59,948 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=42 updating hbase:meta row=cd8331e491339ba2021603869fc044dc, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:31:59,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=43, resume processing ppid=42 2023-09-06T20:31:59,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=43, ppid=42, state=SUCCESS; OpenRegionProcedure cd8331e491339ba2021603869fc044dc, server=jenkins-hbase3.apache.org,36961,1694032287907 in 173 msec 2023-09-06T20:31:59,953 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=42, resume processing ppid=41 2023-09-06T20:31:59,953 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=42, ppid=41, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16940322964961, region=cd8331e491339ba2021603869fc044dc, ASSIGN in 331 msec 2023-09-06T20:31:59,953 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16940322964961","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032319953"}]},"ts":"1694032319953"} 2023-09-06T20:31:59,955 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16940322964961, state=ENABLED in hbase:meta 2023-09-06T20:31:59,957 DEBUG [PEWorker-3 {}] procedure.TruncateTableProcedure(145): truncate 'ns2:test-16940322964961' completed 2023-09-06T20:31:59,958 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=41, state=SUCCESS; TruncateTableProcedure (table=ns2:test-16940322964961 preserveSplits=true) in 1.2240 sec 2023-09-06T20:32:00,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=41 2023-09-06T20:32:00,851 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin$TableFuture(3677): Operation: TRUNCATE, Table Name: ns2:test-16940322964961, procId: 41 completed 2023-09-06T20:32:00,859 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:39327/backupUT/backup_1694032303765/ns2/test-16940322964961/archive/data/ns2/test-16940322964961/1932330ab1e654ad2ac9ae28fd5fcdb3 from hbase tables ns2:test-16940322964961 to tables ns2:test-16940322964961 2023-09-06T20:32:00,859 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-16940322964961 into ns2:test-16940322964961 2023-09-06T20:32:00,862 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-test-16940322964961-1694032320859 from hdfs://localhost:39327/backupUT/backup_1694032303765/ns2/test-16940322964961/archive/data/ns2/test-16940322964961/1932330ab1e654ad2ac9ae28fd5fcdb3 2023-09-06T20:32:00,865 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7c122082 to 127.0.0.1:53515 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:32:00,871 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b1db13e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:00,875 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:32:00,877 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41096, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:32:00,891 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-09-06T20:32:00,892 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16940322964961 2023-09-06T20:32:00,897 DEBUG [hconnection-0x510b2490-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:32:00,898 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52820, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:32:00,902 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16940322964961: 2023-09-06T20:32:00,902 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-09-06T20:32:00,903 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/partitions_71211385-4aa6-4f45-8511-14f7031dfc13 2023-09-06T20:32:01,342 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:01,343 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:01,343 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:01,344 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:02,343 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop-2524219258535147845.jar 2023-09-06T20:32:02,343 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:02,344 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:02,344 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:02,345 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:02,345 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:02,346 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:02,346 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:02,346 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:02,347 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:02,348 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:32:02,348 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:32:02,349 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:32:02,349 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:32:02,350 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:32:02,351 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:32:02,351 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:32:02,352 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:32:02,352 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:32:02,353 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:32:02,353 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:32:02,354 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:32:02,357 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:02,358 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:02,359 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:02,359 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:02,359 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:02,360 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:02,361 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:32:02,361 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:test-16940322964961 2023-09-06T20:32:02,361 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:32:02,362 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7c122082 to 127.0.0.1:53515 2023-09-06T20:32:02,362 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:32:02,362 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-09-06T20:32:02,362 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-09-06T20:32:02,363 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:32:02,368 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-09-06T20:32:02,375 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-09-06T20:32:02,485 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032322445/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:32:02,485 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032322445/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:32:02,568 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:39327/backupUT/backup_1694032303765/ns2/test-16940322964961/archive/data/ns2/test-16940322964961/1932330ab1e654ad2ac9ae28fd5fcdb3/f/c196aaa261e74a059e3eb2c060e5b44a 2023-09-06T20:32:02,574 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-09-06T20:32:02,707 DEBUG [pool-202-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x036eddf9 to 127.0.0.1:53515 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:32:02,715 DEBUG [pool-202-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@82d618, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:02,717 DEBUG [hconnection-0x64d990e9-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:32:02,719 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52822, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:32:02,725 DEBUG [pool-202-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x036eddf9 to 127.0.0.1:53515 2023-09-06T20:32:02,725 DEBUG [pool-202-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:32:02,725 DEBUG [pool-202-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-09-06T20:32:02,726 DEBUG [pool-202-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-09-06T20:32:02,776 DEBUG [pool-202-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-test-16940322964961-1694032320859/_temporary/0/_temporary/attempt_local707137833_0003_r_000000_0/f/4efc6a89488a4cd6a3a32be13544bacd is 37, key is row10/f:q1/1694032300351/Put/seqid=0 2023-09-06T20:32:02,779 WARN [Thread-797 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:36961] Specified favored nodes: [jenkins-hbase3.apache.org:36961] 2023-09-06T20:32:03,554 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-test-16940322964961-1694032320859 2023-09-06T20:32:03,556 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x47ee7c66 to 127.0.0.1:53515 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:32:03,561 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1afe434, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:03,563 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:32:03,565 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52826, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:32:03,575 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-test-16940322964961-1694032320859/_SUCCESS 2023-09-06T20:32:03,580 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:32:03,582 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41110, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:32:03,628 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-test-16940322964961-1694032320859/f/4efc6a89488a4cd6a3a32be13544bacd first=Optional[row0] last=Optional[row98] 2023-09-06T20:32:03,652 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc., hostname=jenkins-hbase3.apache.org,36961,1694032287907, seqNum=2 for row with hfile group [{f,hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-test-16940322964961-1694032320859/f/4efc6a89488a4cd6a3a32be13544bacd}] 2023-09-06T20:32:03,678 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-test-16940322964961-1694032320859/f/4efc6a89488a4cd6a3a32be13544bacd for inclusion in cd8331e491339ba2021603869fc044dc/f 2023-09-06T20:32:03,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-09-06T20:32:03,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(625): Region bounds: first= last= 2023-09-06T20:32:03,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HRegion(2520): Flush status journal for cd8331e491339ba2021603869fc044dc: 2023-09-06T20:32:03,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-test-16940322964961-1694032320859/f/4efc6a89488a4cd6a3a32be13544bacd to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__ns2_test-16940322964961__sc0l919ku4r9oc6n58gvk0mkhem8qt5dn92lehcjs24fahmvtn1vgorktlepe443/f/4efc6a89488a4cd6a3a32be13544bacd 2023-09-06T20:32:03,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__ns2_test-16940322964961__sc0l919ku4r9oc6n58gvk0mkhem8qt5dn92lehcjs24fahmvtn1vgorktlepe443/f/4efc6a89488a4cd6a3a32be13544bacd as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc/f/5f020f30827940489104cc2b9277b6d8_SeqId_4_ 2023-09-06T20:32:03,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x34aad425 to 127.0.0.1:53515 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:32:03,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c5bc6b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:03,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:32:03,713 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41124, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:32:03,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:32:03,721 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52830, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:32:03,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] impl.BackupSystemTable(408): write bulk load descriptor to backup ns2:test-16940322964961 with 1 entries 2023-09-06T20:32:03,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] impl.BackupSystemTable(1582): writing raw bulk path hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc/f/5f020f30827940489104cc2b9277b6d8_SeqId_4_ for ns2:test-16940322964961 cd8331e491339ba2021603869fc044dc 2023-09-06T20:32:03,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] impl.BackupSystemTable(415): written 1 rows for bulk load of ns2:test-16940322964961 2023-09-06T20:32:03,747 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:32:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x34aad425 to 127.0.0.1:53515 2023-09-06T20:32:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:32:03,749 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__ns2_test-16940322964961__sc0l919ku4r9oc6n58gvk0mkhem8qt5dn92lehcjs24fahmvtn1vgorktlepe443/f/4efc6a89488a4cd6a3a32be13544bacd into cd8331e491339ba2021603869fc044dc/f as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc/f/5f020f30827940489104cc2b9277b6d8_SeqId_4_ - updating store file list. 2023-09-06T20:32:03,758 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc/f/5f020f30827940489104cc2b9277b6d8_SeqId_4_ into cd8331e491339ba2021603869fc044dc/f 2023-09-06T20:32:03,758 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__ns2_test-16940322964961__sc0l919ku4r9oc6n58gvk0mkhem8qt5dn92lehcjs24fahmvtn1vgorktlepe443/f/4efc6a89488a4cd6a3a32be13544bacd into cd8331e491339ba2021603869fc044dc/f (new location: hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc/f/5f020f30827940489104cc2b9277b6d8_SeqId_4_) 2023-09-06T20:32:03,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__ns2_test-16940322964961__sc0l919ku4r9oc6n58gvk0mkhem8qt5dn92lehcjs24fahmvtn1vgorktlepe443/f/4efc6a89488a4cd6a3a32be13544bacd 2023-09-06T20:32:03,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x58c6b892 to 127.0.0.1:53515 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:32:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7798ba1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:03,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:32:03,784 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41130, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-09-06T20:32:03,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:32:03,797 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52838, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-09-06T20:32:03,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] impl.BackupSystemTable(388): write bulk load descriptor to backup ns2:test-16940322964961 with 1 entries 2023-09-06T20:32:03,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] impl.BackupSystemTable(1503): writing done bulk path hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc/f/5f020f30827940489104cc2b9277b6d8_SeqId_4_ for ns2:test-16940322964961 cd8331e491339ba2021603869fc044dc 2023-09-06T20:32:03,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] impl.BackupSystemTable(394): written 1 rows for bulk load of ns2:test-16940322964961 2023-09-06T20:32:03,820 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:32:03,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x58c6b892 to 127.0.0.1:53515 2023-09-06T20:32:03,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:32:03,832 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:32:03,833 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x47ee7c66 to 127.0.0.1:53515 2023-09-06T20:32:03,833 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:32:03,834 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-09-06T20:32:03,834 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-09-06T20:32:03,834 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(233): Backup: backup_1694032303765 hdfs://localhost:39327/backupUT/backup_1694032303765/ns2/test-16940322964961/ 2023-09-06T20:32:03,835 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:39327/backupUT/backup_1694032303765/.backup.manifest 2023-09-06T20:32:03,838 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032303765/.backup.manifest 2023-09-06T20:32:03,838 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(148): Restoring 'ns3:test-16940322964962' to 'ns3:test-16940322964962' from full backup image hdfs://localhost:39327/backupUT/backup_1694032303765/ns3/test-16940322964962 2023-09-06T20:32:03,846 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(101): Folder tableArchivePath: hdfs://localhost:39327/backupUT/backup_1694032303765/ns3/test-16940322964962/archive/data/ns3/test-16940322964962 does not exists 2023-09-06T20:32:03,846 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(326): find table descriptor but no archive dir for table ns3:test-16940322964962, will only create table 2023-09-06T20:32:03,848 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(476): Truncating exising target table 'ns3:test-16940322964962', preserving region splits 2023-09-06T20:32:03,849 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin$18(967): Started disable of ns3:test-16940322964962 2023-09-06T20:32:03,849 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable ns3:test-16940322964962 2023-09-06T20:32:03,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=44, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns3:test-16940322964962 2023-09-06T20:32:03,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-09-06T20:32:03,854 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16940322964962","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032323854"}]},"ts":"1694032323854"} 2023-09-06T20:32:03,856 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16940322964962, state=DISABLING in hbase:meta 2023-09-06T20:32:03,859 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set ns3:test-16940322964962 to state=DISABLING 2023-09-06T20:32:03,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:test-16940322964962, region=18ddf30d0a5d8ca482131f8c37827552, UNASSIGN}] 2023-09-06T20:32:03,861 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:test-16940322964962, region=18ddf30d0a5d8ca482131f8c37827552, UNASSIGN 2023-09-06T20:32:03,862 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=45 updating hbase:meta row=18ddf30d0a5d8ca482131f8c37827552, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:03,864 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE; CloseRegionProcedure 18ddf30d0a5d8ca482131f8c37827552, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:32:03,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-09-06T20:32:04,017 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] handler.UnassignRegionHandler(119): Close 18ddf30d0a5d8ca482131f8c37827552 2023-09-06T20:32:04,017 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1668): Closing 18ddf30d0a5d8ca482131f8c37827552, disabling compactions & flushes 2023-09-06T20:32:04,017 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1690): Closing region ns3:test-16940322964962,,1694032300440.18ddf30d0a5d8ca482131f8c37827552. 2023-09-06T20:32:04,017 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16940322964962,,1694032300440.18ddf30d0a5d8ca482131f8c37827552. 2023-09-06T20:32:04,017 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16940322964962,,1694032300440.18ddf30d0a5d8ca482131f8c37827552. after waiting 0 ms 2023-09-06T20:32:04,017 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16940322964962,,1694032300440.18ddf30d0a5d8ca482131f8c37827552. 2023-09-06T20:32:04,022 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/test-16940322964962/18ddf30d0a5d8ca482131f8c37827552/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2023-09-06T20:32:04,022 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:32:04,024 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1909): Closed ns3:test-16940322964962,,1694032300440.18ddf30d0a5d8ca482131f8c37827552. 2023-09-06T20:32:04,024 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1622): Region close journal for 18ddf30d0a5d8ca482131f8c37827552: 2023-09-06T20:32:04,026 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] handler.UnassignRegionHandler(163): Closed 18ddf30d0a5d8ca482131f8c37827552 2023-09-06T20:32:04,026 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=45 updating hbase:meta row=18ddf30d0a5d8ca482131f8c37827552, regionState=CLOSED 2023-09-06T20:32:04,030 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=46, resume processing ppid=45 2023-09-06T20:32:04,030 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=46, ppid=45, state=SUCCESS; CloseRegionProcedure 18ddf30d0a5d8ca482131f8c37827552, server=jenkins-hbase3.apache.org,36961,1694032287907 in 164 msec 2023-09-06T20:32:04,032 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=45, resume processing ppid=44 2023-09-06T20:32:04,032 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=45, ppid=44, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16940322964962, region=18ddf30d0a5d8ca482131f8c37827552, UNASSIGN in 170 msec 2023-09-06T20:32:04,032 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16940322964962","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032324032"}]},"ts":"1694032324032"} 2023-09-06T20:32:04,034 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16940322964962, state=DISABLED in hbase:meta 2023-09-06T20:32:04,036 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(305): Set ns3:test-16940322964962 to state=DISABLED 2023-09-06T20:32:04,038 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=44, state=SUCCESS; DisableTableProcedure table=ns3:test-16940322964962 in 188 msec 2023-09-06T20:32:04,157 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:32:04,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-09-06T20:32:04,158 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin$TableFuture(3677): Operation: DISABLE, Table Name: ns3:test-16940322964962, procId: 44 completed 2023-09-06T20:32:04,159 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin$16(847): Started truncating ns3:test-16940322964962 2023-09-06T20:32:04,159 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$6(2480): Client=jenkins//172.31.12.81 truncate ns3:test-16940322964962 2023-09-06T20:32:04,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=47, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=ns3:test-16940322964962 preserveSplits=true) 2023-09-06T20:32:04,162 DEBUG [PEWorker-5 {}] procedure.TruncateTableProcedure(87): waiting for 'ns3:test-16940322964962' regions in transition 2023-09-06T20:32:04,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-09-06T20:32:04,166 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/test-16940322964962/18ddf30d0a5d8ca482131f8c37827552 2023-09-06T20:32:04,169 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/test-16940322964962/18ddf30d0a5d8ca482131f8c37827552/f, FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/test-16940322964962/18ddf30d0a5d8ca482131f8c37827552/recovered.edits] 2023-09-06T20:32:04,178 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/test-16940322964962/18ddf30d0a5d8ca482131f8c37827552/recovered.edits/5.seqid to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/archive/data/ns3/test-16940322964962/18ddf30d0a5d8ca482131f8c37827552/recovered.edits/5.seqid 2023-09-06T20:32:04,178 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/test-16940322964962/18ddf30d0a5d8ca482131f8c37827552 2023-09-06T20:32:04,179 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(293): Archived ns3:test-16940322964962 regions 2023-09-06T20:32:04,185 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns3:test-16940322964962 from hbase:meta 2023-09-06T20:32:04,189 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(387): Removing 'ns3:test-16940322964962' descriptor. 2023-09-06T20:32:04,190 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(377): Removing 'ns3:test-16940322964962' from region states. 2023-09-06T20:32:04,190 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:test-16940322964962,,1694032300440.18ddf30d0a5d8ca482131f8c37827552.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1694032324190"}]},"ts":"9223372036854775807"} 2023-09-06T20:32:04,192 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-06T20:32:04,192 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 18ddf30d0a5d8ca482131f8c37827552, NAME => 'ns3:test-16940322964962,,1694032300440.18ddf30d0a5d8ca482131f8c37827552.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:32:04,192 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(381): Marking 'ns3:test-16940322964962' as deleted. 2023-09-06T20:32:04,192 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:test-16940322964962","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1694032324192"}]},"ts":"9223372036854775807"} 2023-09-06T20:32:04,195 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1678): Deleted table ns3:test-16940322964962 state from META 2023-09-06T20:32:04,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-09-06T20:32:04,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-09-06T20:32:04,524 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-09-06T20:32:04,525 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41136, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-09-06T20:32:04,616 INFO [RegionOpenAndInit-ns3:test-16940322964962-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 685fcf26b1065c7882673d26a6d8d168, NAME => 'ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16940322964962', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:32:04,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-09-06T20:32:05,023 DEBUG [RegionOpenAndInit-ns3:test-16940322964962-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:32:05,023 DEBUG [RegionOpenAndInit-ns3:test-16940322964962-pool-0 {}] regionserver.HRegion(1668): Closing 685fcf26b1065c7882673d26a6d8d168, disabling compactions & flushes 2023-09-06T20:32:05,023 INFO [RegionOpenAndInit-ns3:test-16940322964962-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168. 2023-09-06T20:32:05,023 DEBUG [RegionOpenAndInit-ns3:test-16940322964962-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168. 2023-09-06T20:32:05,023 DEBUG [RegionOpenAndInit-ns3:test-16940322964962-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168. after waiting 0 ms 2023-09-06T20:32:05,023 DEBUG [RegionOpenAndInit-ns3:test-16940322964962-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168. 2023-09-06T20:32:05,023 INFO [RegionOpenAndInit-ns3:test-16940322964962-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168. 2023-09-06T20:32:05,023 DEBUG [RegionOpenAndInit-ns3:test-16940322964962-pool-0 {}] regionserver.HRegion(1622): Region close journal for 685fcf26b1065c7882673d26a6d8d168: 2023-09-06T20:32:05,025 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1694032325025"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032325025"}]},"ts":"1694032325025"} 2023-09-06T20:32:05,027 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:32:05,028 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16940322964962","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032325027"}]},"ts":"1694032325027"} 2023-09-06T20:32:05,029 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16940322964962, state=ENABLING in hbase:meta 2023-09-06T20:32:05,033 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16940322964962, region=685fcf26b1065c7882673d26a6d8d168, ASSIGN}] 2023-09-06T20:32:05,035 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16940322964962, region=685fcf26b1065c7882673d26a6d8d168, ASSIGN 2023-09-06T20:32:05,036 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16940322964962, region=685fcf26b1065c7882673d26a6d8d168, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36961,1694032287907; forceNewPlan=false, retain=false 2023-09-06T20:32:05,187 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=48 updating hbase:meta row=685fcf26b1065c7882673d26a6d8d168, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:05,190 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE; OpenRegionProcedure 685fcf26b1065c7882673d26a6d8d168, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:32:05,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-09-06T20:32:05,347 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(130): Open ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168. 2023-09-06T20:32:05,347 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7254): Opening region: {ENCODED => 685fcf26b1065c7882673d26a6d8d168, NAME => 'ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:32:05,348 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:32:05,348 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16940322964962 685fcf26b1065c7882673d26a6d8d168 2023-09-06T20:32:05,348 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(888): Instantiated ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:32:05,348 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7296): checking encryption for 685fcf26b1065c7882673d26a6d8d168 2023-09-06T20:32:05,348 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7299): checking classloading for 685fcf26b1065c7882673d26a6d8d168 2023-09-06T20:32:05,354 INFO [StoreOpener-685fcf26b1065c7882673d26a6d8d168-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 685fcf26b1065c7882673d26a6d8d168 2023-09-06T20:32:05,357 INFO [StoreOpener-685fcf26b1065c7882673d26a6d8d168-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 685fcf26b1065c7882673d26a6d8d168 columnFamilyName f 2023-09-06T20:32:05,357 DEBUG [StoreOpener-685fcf26b1065c7882673d26a6d8d168-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:32:05,358 INFO [StoreOpener-685fcf26b1065c7882673d26a6d8d168-1 {}] regionserver.HStore(324): Store=685fcf26b1065c7882673d26a6d8d168/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:32:05,359 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/test-16940322964962/685fcf26b1065c7882673d26a6d8d168 2023-09-06T20:32:05,359 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/test-16940322964962/685fcf26b1065c7882673d26a6d8d168 2023-09-06T20:32:05,362 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1076): writing seq id for 685fcf26b1065c7882673d26a6d8d168 2023-09-06T20:32:05,365 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/test-16940322964962/685fcf26b1065c7882673d26a6d8d168/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:32:05,366 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): Opened 685fcf26b1065c7882673d26a6d8d168; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9927032000, jitterRate=-0.07547309994697571}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:32:05,367 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(992): Region open journal for 685fcf26b1065c7882673d26a6d8d168: 2023-09-06T20:32:05,368 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168., pid=49, masterSystemTime=1694032325342 2023-09-06T20:32:05,370 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168. 2023-09-06T20:32:05,370 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(158): Opened ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168. 2023-09-06T20:32:05,371 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=48 updating hbase:meta row=685fcf26b1065c7882673d26a6d8d168, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:05,374 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=49, resume processing ppid=48 2023-09-06T20:32:05,374 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=49, ppid=48, state=SUCCESS; OpenRegionProcedure 685fcf26b1065c7882673d26a6d8d168, server=jenkins-hbase3.apache.org,36961,1694032287907 in 182 msec 2023-09-06T20:32:05,375 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=48, resume processing ppid=47 2023-09-06T20:32:05,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=48, ppid=47, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16940322964962, region=685fcf26b1065c7882673d26a6d8d168, ASSIGN in 341 msec 2023-09-06T20:32:05,376 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16940322964962","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032325375"}]},"ts":"1694032325375"} 2023-09-06T20:32:05,377 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16940322964962, state=ENABLED in hbase:meta 2023-09-06T20:32:05,382 DEBUG [PEWorker-1 {}] procedure.TruncateTableProcedure(145): truncate 'ns3:test-16940322964962' completed 2023-09-06T20:32:05,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=47, state=SUCCESS; TruncateTableProcedure (table=ns3:test-16940322964962 preserveSplits=true) in 1.2230 sec 2023-09-06T20:32:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-09-06T20:32:06,270 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin$TableFuture(3677): Operation: TRUNCATE, Table Name: ns3:test-16940322964962, procId: 47 completed 2023-09-06T20:32:06,273 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-09-06T20:32:06,274 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(233): Backup: backup_1694032303765 hdfs://localhost:39327/backupUT/backup_1694032303765/ns3/test-16940322964962/ 2023-09-06T20:32:06,274 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-09-06T20:32:06,292 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultipleOverwrite Thread=598 (was 583) Potentially hanging thread: PacketResponder: BP-1247140837-172.31.12.81-1694032281151:blk_1073741856_1032, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x77470b9a-shared-pool-16 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1f412458-shared-pool-6 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x77470b9a-shared-pool-14 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-508974552_16 at /127.0.0.1:47024 [Receiving block BP-1247140837-172.31.12.81-1694032281151:blk_1073741855_1031] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-623328790_16 at /127.0.0.1:39604 [Waiting for operation #25] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: (jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: HFileArchiver-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-508974552_16 at /127.0.0.1:47034 [Receiving block BP-1247140837-172.31.12.81-1694032281151:blk_1073741856_1032] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x77470b9a-shared-pool-12 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/cluster_26d71070-e599-b36c-2d51-b2ad8bb6051e/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/cluster_26d71070-e599-b36c-2d51-b2ad8bb6051e/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: HFileArchiver-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1247140837-172.31.12.81-1694032281151:blk_1073741855_1031, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x77470b9a-shared-pool-11 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x77470b9a-shared-pool-15 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x77470b9a-shared-pool-13 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-508974552_16 at /127.0.0.1:52406 [Waiting for operation #5] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=669 (was 660) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=259 (was 229) - SystemLoadAverage LEAK? -, ProcessCount=167 (was 167), AvailableMemoryMB=3703 (was 3757) 2023-09-06T20:32:06,292 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(130): Thread=598 is superior to 500 2023-09-06T20:32:06,307 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingleOverwrite Thread=598, OpenFileDescriptor=669, MaxFileDescriptor=60000, SystemLoadAverage=259, ProcessCount=167, AvailableMemoryMB=3702 2023-09-06T20:32:06,307 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(130): Thread=598 is superior to 500 2023-09-06T20:32:06,307 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] backup.TestFullRestore(173): test full restore on a single table empty table 2023-09-06T20:32:06,308 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x66f5ac7e to 127.0.0.1:53515 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:32:06,314 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@664e2fa3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:06,316 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:32:06,318 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52848, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:32:06,320 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:32:06,321 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41148, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:32:06,335 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:32:06,340 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:32:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032326340 (current time:1694032326340). 2023-09-06T20:32:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:32:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:32:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x270f03b7 to 127.0.0.1:53515 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:32:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d0d056e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:32:06,353 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52860, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:32:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x270f03b7 to 127.0.0.1:53515 2023-09-06T20:32:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:32:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:32:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=50, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:32:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 50 2023-09-06T20:32:06,358 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:32:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-09-06T20:32:06,359 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:32:06,361 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:32:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-09-06T20:32:06,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-09-06T20:32:06,782 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:32:06,782 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281}] 2023-09-06T20:32:06,784 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 2023-09-06T20:32:06,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=51 2023-09-06T20:32:06,936 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:32:06,936 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2819): Flushing cf63a85c53279bc94811b50578f31281 2/2 column families, dataSize=1.51 KB heapSize=2.81 KB 2023-09-06T20:32:06,951 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/0379b1e1f59e40d2ac2991b2ae6b86ce is 171, key is trslm:hdfs://localhost:39327/backupUT\x00ns2:test-16940322964961/meta:log-roll-map/1694032317434/Put/seqid=0 2023-09-06T20:32:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-09-06T20:32:07,356 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=792 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/0379b1e1f59e40d2ac2991b2ae6b86ce 2023-09-06T20:32:07,367 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/baca9a30045a4dabad9b3b78a8fd6c86 is 458, key is session:backup_1694032303765/session:context/1694032318313/Put/seqid=0 2023-09-06T20:32:07,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-09-06T20:32:07,772 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=754 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/baca9a30045a4dabad9b3b78a8fd6c86 2023-09-06T20:32:07,781 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/0379b1e1f59e40d2ac2991b2ae6b86ce as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/0379b1e1f59e40d2ac2991b2ae6b86ce 2023-09-06T20:32:07,789 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/0379b1e1f59e40d2ac2991b2ae6b86ce, entries=6, sequenceid=17, filesize=5.8 K 2023-09-06T20:32:07,791 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/baca9a30045a4dabad9b3b78a8fd6c86 as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/baca9a30045a4dabad9b3b78a8fd6c86 2023-09-06T20:32:07,796 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/baca9a30045a4dabad9b3b78a8fd6c86, entries=2, sequenceid=17, filesize=5.4 K 2023-09-06T20:32:07,797 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(3022): Finished flush of dataSize ~1.51 KB/1546, heapSize ~2.78 KB/2848, currentSize=0 B/0 for cf63a85c53279bc94811b50578f31281 in 861ms, sequenceid=17, compaction requested=false 2023-09-06T20:32:07,797 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2520): Flush status journal for cf63a85c53279bc94811b50578f31281: 2023-09-06T20:32:07,798 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. for snapshot_backup_system completed. 2023-09-06T20:32:07,798 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:32:07,798 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:32:07,798 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/0379b1e1f59e40d2ac2991b2ae6b86ce] hfiles 2023-09-06T20:32:07,798 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/0379b1e1f59e40d2ac2991b2ae6b86ce for snapshot=snapshot_backup_system 2023-09-06T20:32:07,798 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/4ced064cdbd34bd6b594c76f049fa4ee, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/baca9a30045a4dabad9b3b78a8fd6c86] hfiles 2023-09-06T20:32:07,798 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/4ced064cdbd34bd6b594c76f049fa4ee for snapshot=snapshot_backup_system 2023-09-06T20:32:07,798 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/baca9a30045a4dabad9b3b78a8fd6c86 for snapshot=snapshot_backup_system 2023-09-06T20:32:08,204 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:32:08,204 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=51 2023-09-06T20:32:08,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=51 2023-09-06T20:32:08,205 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region cf63a85c53279bc94811b50578f31281 2023-09-06T20:32:08,206 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 2023-09-06T20:32:08,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=51, resume processing ppid=50 2023-09-06T20:32:08,210 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:32:08,210 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=51, ppid=50, state=SUCCESS; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 in 1.4250 sec 2023-09-06T20:32:08,211 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:32:08,211 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:32:08,211 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:32:08,212 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:32:08,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-09-06T20:32:08,628 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:32:08,636 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:32:08,636 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:32:08,638 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:32:08,638 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 50 2023-09-06T20:32:08,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=50, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2830 sec 2023-09-06T20:32:09,525 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:32:10,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-09-06T20:32:10,467 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: backup:system, procId: 50 completed 2023-09-06T20:32:10,467 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(115): Backup backup_1694032326314 started at 1694032330467. 2023-09-06T20:32:10,469 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(119): Backup session backup_1694032326314 has been started. 2023-09-06T20:32:10,475 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-09-06T20:32:10,475 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-09-06T20:32:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-09-06T20:32:10,476 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-09-06T20:32:10,476 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-09-06T20:32:10,477 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-09-06T20:32:10,477 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-09-06T20:32:10,477 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-09-06T20:32:10,477 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:32:10,479 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:10,479 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-09-06T20:32:10,479 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:32:10,479 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:32:10,479 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:10,479 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-09-06T20:32:10,479 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-09-06T20:32:10,480 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-09-06T20:32:10,480 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-09-06T20:32:10,480 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:32:10,480 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-09-06T20:32:10,480 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-09-06T20:32:10,480 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-09-06T20:32:10,480 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-09-06T20:32:10,480 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-09-06T20:32:10,481 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-09-06T20:32:10,481 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-09-06T20:32:10,481 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,36961,1694032287907' joining acquired barrier for procedure (rolllog) in zk 2023-09-06T20:32:10,485 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-09-06T20:32:10,485 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:10,485 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:10,485 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:32:10,485 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:32:10,485 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-09-06T20:32:10,485 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-09-06T20:32:10,485 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:32:10,486 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:32:10,486 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:32:10,486 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:10,487 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:32:10,487 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,36961,1694032287907' joining acquired barrier for procedure 'rolllog' on coordinator 2023-09-06T20:32:10,487 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@7e2bfc9[Count = 0] remaining members to acquire global barrier 2023-09-06T20:32:10,487 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-09-06T20:32:10,487 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-09-06T20:32:10,488 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-09-06T20:32:10,488 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-09-06T20:32:10,488 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-09-06T20:32:10,489 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-09-06T20:32:10,489 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:10,489 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-09-06T20:32:10,489 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-09-06T20:32:10,489 DEBUG [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:10,489 INFO [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1694032308487 highest: 1694032308487 on jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:10,489 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta:.meta(num 1694032308462) roll requested 2023-09-06T20:32:10,501 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032330489.meta, exclude list is [], retry=0 2023-09-06T20:32:10,505 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK] 2023-09-06T20:32:10,507 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032308462.meta with entries=24, filesize=6.26 KB; new WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032330489.meta 2023-09-06T20:32:10,507 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK]] 2023-09-06T20:32:10,507 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032308462.meta is not closed yet, will try archiving it next time 2023-09-06T20:32:10,507 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36961%2C1694032287907:(num 1694032308487) roll requested 2023-09-06T20:32:10,518 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032330507, exclude list is [], retry=0 2023-09-06T20:32:10,522 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK] 2023-09-06T20:32:10,523 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032308487 with entries=19, filesize=6.26 KB; new WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032330507 2023-09-06T20:32:10,524 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK]] 2023-09-06T20:32:10,524 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032308487 is not closed yet, will try archiving it next time 2023-09-06T20:32:10,529 DEBUG [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-09-06T20:32:10,529 INFO [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1694032330507 on jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:10,545 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-09-06T20:32:10,545 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,36961,1694032287907' in zk 2023-09-06T20:32:10,548 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:10,548 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-09-06T20:32:10,548 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:32:10,548 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-09-06T20:32:10,548 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:10,549 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:32:10,549 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:32:10,550 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:32:10,551 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:32:10,551 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:32:10,552 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:10,552 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:32:10,553 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:32:10,553 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:10,554 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,36961,1694032287907': 2023-09-06T20:32:10,554 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,36961,1694032287907' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-09-06T20:32:10,554 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-09-06T20:32:10,554 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-09-06T20:32:10,554 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-09-06T20:32:10,554 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-09-06T20:32:10,554 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-09-06T20:32:10,556 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-09-06T20:32:10,556 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-09-06T20:32:10,556 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-09-06T20:32:10,556 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-09-06T20:32:10,556 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:32:10,556 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-09-06T20:32:10,556 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:32:10,556 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:32:10,556 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:32:10,556 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:32:10,558 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:32:10,558 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:10,558 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:32:10,558 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:32:10,559 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:32:10,559 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:32:10,560 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:10,560 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:10,562 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:32:10,563 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:32:10,564 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:10,569 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:10,569 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-09-06T20:32:10,569 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-09-06T20:32:10,569 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:10,569 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:32:10,569 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-09-06T20:32:10,569 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-09-06T20:32:10,570 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-09-06T20:32:10,570 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-09-06T20:32:10,570 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-09-06T20:32:10,570 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-09-06T20:32:10,570 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-09-06T20:32:10,570 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:32:10,570 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:32:10,570 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2023-09-06T20:32:10,570 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin(2838): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-09-06T20:32:10,571 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin(2846): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-09-06T20:32:10,571 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:32:10,571 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:32:10,671 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin(2852): Getting current status of procedure from master... 2023-09-06T20:32:10,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-09-06T20:32:10,676 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1694032330675_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } 2023-09-06T20:32:10,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032330676 (current time:1694032330676). 2023-09-06T20:32:10,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:32:10,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1694032330675_default_test-1694032296496 VERSION not specified, setting to 2 2023-09-06T20:32:10,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x528a784a to 127.0.0.1:53515 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:32:10,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67192100, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:10,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:32:10,691 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41250, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:32:10,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x528a784a to 127.0.0.1:53515 2023-09-06T20:32:10,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:32:10,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:32:10,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=52, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1694032330675_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } 2023-09-06T20:32:10,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1694032330675_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-09-06T20:32:10,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-09-06T20:32:10,696 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1694032330675_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:32:10,698 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1694032330675_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:32:10,700 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1694032330675_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:32:10,713 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1694032330675_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:32:10,713 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure 98558d33a94bee7957598a35e3cbc370}] 2023-09-06T20:32:10,715 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure 98558d33a94bee7957598a35e3cbc370 2023-09-06T20:32:10,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-09-06T20:32:10,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=53 2023-09-06T20:32:10,867 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1694032296496,,1694032296981.98558d33a94bee7957598a35e3cbc370. 2023-09-06T20:32:10,869 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2819): Flushing 98558d33a94bee7957598a35e3cbc370 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-09-06T20:32:10,889 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/98558d33a94bee7957598a35e3cbc370/.tmp/f/f0d99ccf76014f5c8e0fbba44b1cd825 is 37, key is row10/f:q1/1694032299136/Put/seqid=0 2023-09-06T20:32:10,907 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/98558d33a94bee7957598a35e3cbc370/.tmp/f/f0d99ccf76014f5c8e0fbba44b1cd825 2023-09-06T20:32:10,916 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/98558d33a94bee7957598a35e3cbc370/.tmp/f/f0d99ccf76014f5c8e0fbba44b1cd825 as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/98558d33a94bee7957598a35e3cbc370/f/f0d99ccf76014f5c8e0fbba44b1cd825 2023-09-06T20:32:10,924 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/98558d33a94bee7957598a35e3cbc370/f/f0d99ccf76014f5c8e0fbba44b1cd825, entries=99, sequenceid=103, filesize=8.2 K 2023-09-06T20:32:10,925 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 98558d33a94bee7957598a35e3cbc370 in 57ms, sequenceid=103, compaction requested=false 2023-09-06T20:32:10,927 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2520): Flush status journal for 98558d33a94bee7957598a35e3cbc370: 2023-09-06T20:32:10,927 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1694032296496,,1694032296981.98558d33a94bee7957598a35e3cbc370. for snapshot_1694032330675_default_test-1694032296496 completed. 2023-09-06T20:32:10,927 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(240): Storing 'test-1694032296496,,1694032296981.98558d33a94bee7957598a35e3cbc370.' region-info for snapshot=snapshot_1694032330675_default_test-1694032296496 2023-09-06T20:32:10,927 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:32:10,927 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/98558d33a94bee7957598a35e3cbc370/f/f0d99ccf76014f5c8e0fbba44b1cd825] hfiles 2023-09-06T20:32:10,927 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/98558d33a94bee7957598a35e3cbc370/f/f0d99ccf76014f5c8e0fbba44b1cd825 for snapshot=snapshot_1694032330675_default_test-1694032296496 2023-09-06T20:32:10,939 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1694032296496,,1694032296981.98558d33a94bee7957598a35e3cbc370. 2023-09-06T20:32:10,939 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=53 2023-09-06T20:32:10,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=53 2023-09-06T20:32:10,939 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1694032330675_default_test-1694032296496 on region 98558d33a94bee7957598a35e3cbc370 2023-09-06T20:32:10,941 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure 98558d33a94bee7957598a35e3cbc370 2023-09-06T20:32:10,944 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=53, resume processing ppid=52 2023-09-06T20:32:10,944 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=53, ppid=52, state=SUCCESS; SnapshotRegionProcedure 98558d33a94bee7957598a35e3cbc370 in 229 msec 2023-09-06T20:32:10,944 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1694032330675_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:32:10,945 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1694032330675_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:32:10,946 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1694032330675_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:32:10,946 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1694032330675_default_test-1694032296496 2023-09-06T20:32:10,947 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032330675_default_test-1694032296496 2023-09-06T20:32:11,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-09-06T20:32:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-09-06T20:32:11,365 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1694032330675_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:32:11,371 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1694032330675_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:32:11,372 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032330675_default_test-1694032296496 to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032330675_default_test-1694032296496 2023-09-06T20:32:11,373 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1694032330675_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:32:11,373 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1694032330675_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-09-06T20:32:11,374 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=52, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1694032330675_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } in 680 msec 2023-09-06T20:32:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-09-06T20:32:11,807 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: default:test-1694032296496, procId: 52 completed 2023-09-06T20:32:11,807 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1694032326314 2023-09-06T20:32:11,807 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-09-06T20:32:11,807 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-09-06T20:32:11,807 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1694032326314_test-1694032296496 2023-09-06T20:32:11,807 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1694032330675_default_test-1694032296496 to hdfs://localhost:39327/backupUT/backup_1694032326314/default/test-1694032296496/ 2023-09-06T20:32:11,807 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-09-06T20:32:11,831 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:32:11,831 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37419,1694032284546_551324602_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:39327/backupUT/backup_1694032326314/default/test-1694032296496, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:39327/backupUT/backup_1694032326314/default/test-1694032296496/.hbase-snapshot/.tmp/snapshot_1694032330675_default_test-1694032296496 2023-09-06T20:32:11,831 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1, snapshotDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032330675_default_test-1694032296496. 2023-09-06T20:32:11,843 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032330675_default_test-1694032296496 to hdfs://localhost:39327/backupUT/backup_1694032326314/default/test-1694032296496/.hbase-snapshot/.tmp/snapshot_1694032330675_default_test-1694032296496 2023-09-06T20:32:12,258 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:12,259 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:12,260 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:12,260 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:13,241 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-09-06T20:32:13,241 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-09-06T20:32:13,252 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop-8509768729060205779.jar 2023-09-06T20:32:13,252 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:13,253 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:13,253 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:13,254 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:13,254 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:13,254 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:13,255 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:13,255 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:13,256 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:13,256 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:32:13,257 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:32:13,257 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:32:13,258 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:32:13,258 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:32:13,259 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:32:13,259 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:32:13,259 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:32:13,260 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:32:13,260 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:32:13,261 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:32:13,261 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:32:13,262 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:32:13,262 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:32:13,263 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:32:13,263 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:32:13,264 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:32:13,264 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:32:13,265 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:32:13,300 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-09-06T20:32:13,319 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1694032330675_default_test-1694032296496' hfile list 2023-09-06T20:32:13,322 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-09-06T20:32:13,395 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032333360/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:32:13,395 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032333360/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:32:13,449 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-09-06T20:32:13,474 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/98558d33a94bee7957598a35e3cbc370/f/f0d99ccf76014f5c8e0fbba44b1cd825 output=hdfs://localhost:39327/backupUT/backup_1694032326314/default/test-1694032296496/archive/data/default/test-1694032296496/98558d33a94bee7957598a35e3cbc370/f/f0d99ccf76014f5c8e0fbba44b1cd825 2023-09-06T20:32:13,474 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 4.004M/sec 2023-09-06T20:32:14,431 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-09-06T20:32:14,432 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-09-06T20:32:14,439 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1694032330675_default_test-1694032296496 2023-09-06T20:32:14,439 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1694032330675_default_test-1694032296496 finished. 2023-09-06T20:32:14,439 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1014): test-1694032296496 2023-09-06T20:32:14,446 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1694032326314 2023-09-06T20:32:14,446 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-09-06T20:32:14,852 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:39327/backupUT/backup_1694032326314/.backup.manifest 2023-09-06T20:32:14,852 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(358): Backup backup_1694032326314 finished: type=FULL,tablelist=test-1694032296496,targetRootDir=hdfs://localhost:39327/backupUT,startts=1694032330467,completets=1694032334446,bytescopied=0 2023-09-06T20:32:14,852 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-09-06T20:32:14,852 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1694032330675_default_test-1694032296496 2023-09-06T20:32:14,853 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1694032330675_default_test-1694032296496" 2023-09-06T20:32:14,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1694032330675_default_test-1694032296496 2023-09-06T20:32:14,857 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1694032330675_default_test-1694032296496 for backup backup_1694032326314 succeeded. 2023-09-06T20:32:14,858 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:32:14,863 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:32:14,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:32:14,865 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:32:14,867 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:32:14,873 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(376): Backup backup_1694032326314 completed. 2023-09-06T20:32:14,873 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:32:14,873 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x66f5ac7e to 127.0.0.1:53515 2023-09-06T20:32:14,873 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:32:14,885 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] backup.TestFullRestore(178): backup complete 2023-09-06T20:32:14,889 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:39327/backupUT/backup_1694032326314/.backup.manifest 2023-09-06T20:32:14,891 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032326314/.backup.manifest 2023-09-06T20:32:14,891 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:39327/backupUT/backup_1694032326314/.backup.manifest 2023-09-06T20:32:14,893 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032326314/.backup.manifest 2023-09-06T20:32:14,893 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(148): Restoring 'test-1694032296496' to 'test-1694032296496' from full backup image hdfs://localhost:39327/backupUT/backup_1694032326314/default/test-1694032296496 2023-09-06T20:32:14,901 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(476): Truncating exising target table 'test-1694032296496', preserving region splits 2023-09-06T20:32:14,901 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin$18(967): Started disable of test-1694032296496 2023-09-06T20:32:14,902 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable test-1694032296496 2023-09-06T20:32:14,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=test-1694032296496 2023-09-06T20:32:14,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-09-06T20:32:14,906 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1694032296496","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032334906"}]},"ts":"1694032334906"} 2023-09-06T20:32:14,907 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1694032296496, state=DISABLING in hbase:meta 2023-09-06T20:32:14,910 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(293): Set test-1694032296496 to state=DISABLING 2023-09-06T20:32:14,911 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1694032296496, region=98558d33a94bee7957598a35e3cbc370, UNASSIGN}] 2023-09-06T20:32:14,913 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1694032296496, region=98558d33a94bee7957598a35e3cbc370, UNASSIGN 2023-09-06T20:32:14,913 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=98558d33a94bee7957598a35e3cbc370, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:14,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE; CloseRegionProcedure 98558d33a94bee7957598a35e3cbc370, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:32:15,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-09-06T20:32:15,067 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] handler.UnassignRegionHandler(119): Close 98558d33a94bee7957598a35e3cbc370 2023-09-06T20:32:15,067 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1668): Closing 98558d33a94bee7957598a35e3cbc370, disabling compactions & flushes 2023-09-06T20:32:15,067 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1690): Closing region test-1694032296496,,1694032296981.98558d33a94bee7957598a35e3cbc370. 2023-09-06T20:32:15,067 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1694032296496,,1694032296981.98558d33a94bee7957598a35e3cbc370. 2023-09-06T20:32:15,067 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1778): Acquired close lock on test-1694032296496,,1694032296981.98558d33a94bee7957598a35e3cbc370. after waiting 0 ms 2023-09-06T20:32:15,067 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1788): Updates disabled for region test-1694032296496,,1694032296981.98558d33a94bee7957598a35e3cbc370. 2023-09-06T20:32:15,073 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/98558d33a94bee7957598a35e3cbc370/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-09-06T20:32:15,074 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:32:15,075 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1909): Closed test-1694032296496,,1694032296981.98558d33a94bee7957598a35e3cbc370. 2023-09-06T20:32:15,075 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1622): Region close journal for 98558d33a94bee7957598a35e3cbc370: 2023-09-06T20:32:15,077 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] handler.UnassignRegionHandler(163): Closed 98558d33a94bee7957598a35e3cbc370 2023-09-06T20:32:15,077 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=98558d33a94bee7957598a35e3cbc370, regionState=CLOSED 2023-09-06T20:32:15,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=56, resume processing ppid=55 2023-09-06T20:32:15,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=56, ppid=55, state=SUCCESS; CloseRegionProcedure 98558d33a94bee7957598a35e3cbc370, server=jenkins-hbase3.apache.org,36961,1694032287907 in 164 msec 2023-09-06T20:32:15,082 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=55, resume processing ppid=54 2023-09-06T20:32:15,082 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=55, ppid=54, state=SUCCESS; TransitRegionStateProcedure table=test-1694032296496, region=98558d33a94bee7957598a35e3cbc370, UNASSIGN in 169 msec 2023-09-06T20:32:15,083 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1694032296496","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032335083"}]},"ts":"1694032335083"} 2023-09-06T20:32:15,084 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1694032296496, state=DISABLED in hbase:meta 2023-09-06T20:32:15,086 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set test-1694032296496 to state=DISABLED 2023-09-06T20:32:15,088 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=54, state=SUCCESS; DisableTableProcedure table=test-1694032296496 in 185 msec 2023-09-06T20:32:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-09-06T20:32:15,208 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin$TableFuture(3677): Operation: DISABLE, Table Name: default:test-1694032296496, procId: 54 completed 2023-09-06T20:32:15,209 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin$16(847): Started truncating test-1694032296496 2023-09-06T20:32:15,209 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$6(2480): Client=jenkins//172.31.12.81 truncate test-1694032296496 2023-09-06T20:32:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=57, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=test-1694032296496 preserveSplits=true) 2023-09-06T20:32:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=57 2023-09-06T20:32:15,212 DEBUG [PEWorker-1 {}] procedure.TruncateTableProcedure(87): waiting for 'test-1694032296496' regions in transition 2023-09-06T20:32:15,215 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/98558d33a94bee7957598a35e3cbc370 2023-09-06T20:32:15,217 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/98558d33a94bee7957598a35e3cbc370/f, FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/98558d33a94bee7957598a35e3cbc370/recovered.edits] 2023-09-06T20:32:15,221 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/98558d33a94bee7957598a35e3cbc370/f/f0d99ccf76014f5c8e0fbba44b1cd825 to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/archive/data/default/test-1694032296496/98558d33a94bee7957598a35e3cbc370/f/f0d99ccf76014f5c8e0fbba44b1cd825 2023-09-06T20:32:15,225 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/98558d33a94bee7957598a35e3cbc370/recovered.edits/106.seqid to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/archive/data/default/test-1694032296496/98558d33a94bee7957598a35e3cbc370/recovered.edits/106.seqid 2023-09-06T20:32:15,226 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/98558d33a94bee7957598a35e3cbc370 2023-09-06T20:32:15,226 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(293): Archived test-1694032296496 regions 2023-09-06T20:32:15,231 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of test-1694032296496 from hbase:meta 2023-09-06T20:32:15,232 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(387): Removing 'test-1694032296496' descriptor. 2023-09-06T20:32:15,233 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(377): Removing 'test-1694032296496' from region states. 2023-09-06T20:32:15,233 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"test-1694032296496,,1694032296981.98558d33a94bee7957598a35e3cbc370.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1694032335233"}]},"ts":"9223372036854775807"} 2023-09-06T20:32:15,235 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-06T20:32:15,235 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 98558d33a94bee7957598a35e3cbc370, NAME => 'test-1694032296496,,1694032296981.98558d33a94bee7957598a35e3cbc370.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:32:15,235 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(381): Marking 'test-1694032296496' as deleted. 2023-09-06T20:32:15,235 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"test-1694032296496","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1694032335235"}]},"ts":"9223372036854775807"} 2023-09-06T20:32:15,237 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table test-1694032296496 state from META 2023-09-06T20:32:15,248 INFO [RegionOpenAndInit-test-1694032296496-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => f63c4b76a967620ba02f0f167d7de454, NAME => 'test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1694032296496', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:32:15,254 DEBUG [RegionOpenAndInit-test-1694032296496-pool-0 {}] regionserver.HRegion(888): Instantiated test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:32:15,254 DEBUG [RegionOpenAndInit-test-1694032296496-pool-0 {}] regionserver.HRegion(1668): Closing f63c4b76a967620ba02f0f167d7de454, disabling compactions & flushes 2023-09-06T20:32:15,254 INFO [RegionOpenAndInit-test-1694032296496-pool-0 {}] regionserver.HRegion(1690): Closing region test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454. 2023-09-06T20:32:15,254 DEBUG [RegionOpenAndInit-test-1694032296496-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454. 2023-09-06T20:32:15,254 DEBUG [RegionOpenAndInit-test-1694032296496-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454. after waiting 0 ms 2023-09-06T20:32:15,254 DEBUG [RegionOpenAndInit-test-1694032296496-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454. 2023-09-06T20:32:15,254 INFO [RegionOpenAndInit-test-1694032296496-pool-0 {}] regionserver.HRegion(1909): Closed test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454. 2023-09-06T20:32:15,254 DEBUG [RegionOpenAndInit-test-1694032296496-pool-0 {}] regionserver.HRegion(1622): Region close journal for f63c4b76a967620ba02f0f167d7de454: 2023-09-06T20:32:15,255 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1694032335255"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032335255"}]},"ts":"1694032335255"} 2023-09-06T20:32:15,257 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:32:15,257 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1694032296496","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032335257"}]},"ts":"1694032335257"} 2023-09-06T20:32:15,259 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1694032296496, state=ENABLING in hbase:meta 2023-09-06T20:32:15,264 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1694032296496, region=f63c4b76a967620ba02f0f167d7de454, ASSIGN}] 2023-09-06T20:32:15,265 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1694032296496, region=f63c4b76a967620ba02f0f167d7de454, ASSIGN 2023-09-06T20:32:15,266 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1694032296496, region=f63c4b76a967620ba02f0f167d7de454, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36961,1694032287907; forceNewPlan=false, retain=false 2023-09-06T20:32:15,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=57 2023-09-06T20:32:15,418 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=f63c4b76a967620ba02f0f167d7de454, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:15,419 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE; OpenRegionProcedure f63c4b76a967620ba02f0f167d7de454, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:32:15,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=57 2023-09-06T20:32:15,575 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(130): Open test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454. 2023-09-06T20:32:15,575 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7254): Opening region: {ENCODED => f63c4b76a967620ba02f0f167d7de454, NAME => 'test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:32:15,575 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:32:15,575 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1694032296496 f63c4b76a967620ba02f0f167d7de454 2023-09-06T20:32:15,575 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(888): Instantiated test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:32:15,575 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7296): checking encryption for f63c4b76a967620ba02f0f167d7de454 2023-09-06T20:32:15,575 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7299): checking classloading for f63c4b76a967620ba02f0f167d7de454 2023-09-06T20:32:15,577 INFO [StoreOpener-f63c4b76a967620ba02f0f167d7de454-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region f63c4b76a967620ba02f0f167d7de454 2023-09-06T20:32:15,578 INFO [StoreOpener-f63c4b76a967620ba02f0f167d7de454-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f63c4b76a967620ba02f0f167d7de454 columnFamilyName f 2023-09-06T20:32:15,578 DEBUG [StoreOpener-f63c4b76a967620ba02f0f167d7de454-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:32:15,579 INFO [StoreOpener-f63c4b76a967620ba02f0f167d7de454-1 {}] regionserver.HStore(324): Store=f63c4b76a967620ba02f0f167d7de454/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:32:15,580 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454 2023-09-06T20:32:15,580 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454 2023-09-06T20:32:15,582 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1076): writing seq id for f63c4b76a967620ba02f0f167d7de454 2023-09-06T20:32:15,584 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:32:15,584 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): Opened f63c4b76a967620ba02f0f167d7de454; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10448906400, jitterRate=-0.0268697589635849}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:32:15,585 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(992): Region open journal for f63c4b76a967620ba02f0f167d7de454: 2023-09-06T20:32:15,586 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454., pid=59, masterSystemTime=1694032335571 2023-09-06T20:32:15,587 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454. 2023-09-06T20:32:15,587 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(158): Opened test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454. 2023-09-06T20:32:15,588 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=f63c4b76a967620ba02f0f167d7de454, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:15,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=59, resume processing ppid=58 2023-09-06T20:32:15,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=59, ppid=58, state=SUCCESS; OpenRegionProcedure f63c4b76a967620ba02f0f167d7de454, server=jenkins-hbase3.apache.org,36961,1694032287907 in 170 msec 2023-09-06T20:32:15,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=58, resume processing ppid=57 2023-09-06T20:32:15,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=58, ppid=57, state=SUCCESS; TransitRegionStateProcedure table=test-1694032296496, region=f63c4b76a967620ba02f0f167d7de454, ASSIGN in 326 msec 2023-09-06T20:32:15,591 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1694032296496","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032335591"}]},"ts":"1694032335591"} 2023-09-06T20:32:15,593 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1694032296496, state=ENABLED in hbase:meta 2023-09-06T20:32:15,594 DEBUG [PEWorker-2 {}] procedure.TruncateTableProcedure(145): truncate 'test-1694032296496' completed 2023-09-06T20:32:15,595 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=57, state=SUCCESS; TruncateTableProcedure (table=test-1694032296496 preserveSplits=true) in 385 msec 2023-09-06T20:32:15,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=57 2023-09-06T20:32:15,815 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin$TableFuture(3677): Operation: TRUNCATE, Table Name: default:test-1694032296496, procId: 57 completed 2023-09-06T20:32:15,819 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:39327/backupUT/backup_1694032326314/default/test-1694032296496/archive/data/default/test-1694032296496/98558d33a94bee7957598a35e3cbc370 from hbase tables test-1694032296496 to tables test-1694032296496 2023-09-06T20:32:15,819 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1694032296496 into test-1694032296496 2023-09-06T20:32:15,821 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-test-1694032296496-1694032335819 from hdfs://localhost:39327/backupUT/backup_1694032326314/default/test-1694032296496/archive/data/default/test-1694032296496/98558d33a94bee7957598a35e3cbc370 2023-09-06T20:32:15,822 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x61fd49bb to 127.0.0.1:53515 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:32:15,828 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c20aec4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:15,829 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:32:15,830 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45472, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:32:15,831 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-09-06T20:32:15,832 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1694032296496 2023-09-06T20:32:15,834 DEBUG [hconnection-0x78241e74-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:32:15,836 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41252, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:32:15,838 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1694032296496: 2023-09-06T20:32:15,838 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-09-06T20:32:15,838 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/partitions_f7465688-149a-488d-a5b1-3692dba19c9d 2023-09-06T20:32:16,247 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:16,247 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:16,248 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:16,248 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:17,218 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop-7936621162831362171.jar 2023-09-06T20:32:17,218 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:17,218 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:17,219 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:17,219 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:17,220 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:17,220 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:17,220 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:17,221 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:17,221 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:17,222 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:32:17,223 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:32:17,223 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:32:17,223 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:32:17,224 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:32:17,224 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:32:17,224 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:32:17,225 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:32:17,225 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:32:17,226 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:32:17,226 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:32:17,226 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:32:17,227 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:17,227 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:17,228 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:17,228 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:17,228 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:17,229 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:17,229 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:32:17,230 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: test-1694032296496 2023-09-06T20:32:17,230 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:32:17,230 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x61fd49bb to 127.0.0.1:53515 2023-09-06T20:32:17,230 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:32:17,230 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-09-06T20:32:17,230 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-09-06T20:32:17,231 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:32:17,236 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-09-06T20:32:17,242 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-09-06T20:32:17,322 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032337283/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:32:17,322 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032337283/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:32:17,380 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:39327/backupUT/backup_1694032326314/default/test-1694032296496/archive/data/default/test-1694032296496/98558d33a94bee7957598a35e3cbc370/f/f0d99ccf76014f5c8e0fbba44b1cd825 2023-09-06T20:32:17,384 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-09-06T20:32:17,407 DEBUG [pool-236-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x20537aa2 to 127.0.0.1:53515 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:32:17,413 DEBUG [pool-236-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@209feab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:17,414 DEBUG [hconnection-0x71a7f372-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:32:17,416 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41258, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:32:17,420 DEBUG [pool-236-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x20537aa2 to 127.0.0.1:53515 2023-09-06T20:32:17,420 DEBUG [pool-236-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:32:17,420 DEBUG [pool-236-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-09-06T20:32:17,420 DEBUG [pool-236-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-09-06T20:32:17,428 DEBUG [pool-236-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-test-1694032296496-1694032335819/_temporary/0/_temporary/attempt_local749696531_0005_r_000000_0/f/3847e532996147c091bb820ef5a3f90e is 37, key is row10/f:q1/1694032299136/Put/seqid=0 2023-09-06T20:32:17,429 WARN [Thread-952 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:36961] Specified favored nodes: [jenkins-hbase3.apache.org:36961] 2023-09-06T20:32:18,379 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-test-1694032296496-1694032335819 2023-09-06T20:32:18,380 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1bd2beab to 127.0.0.1:53515 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:32:18,410 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@641ba92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:18,412 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:32:18,413 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41272, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:32:18,419 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-test-1694032296496-1694032335819/_SUCCESS 2023-09-06T20:32:18,424 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:32:18,425 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45482, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:32:18,436 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-test-1694032296496-1694032335819/f/3847e532996147c091bb820ef5a3f90e first=Optional[row0] last=Optional[row98] 2023-09-06T20:32:18,438 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454., hostname=jenkins-hbase3.apache.org,36961,1694032287907, seqNum=2 for row with hfile group [{f,hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-test-1694032296496-1694032335819/f/3847e532996147c091bb820ef5a3f90e}] 2023-09-06T20:32:18,443 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-test-1694032296496-1694032335819/f/3847e532996147c091bb820ef5a3f90e for inclusion in f63c4b76a967620ba02f0f167d7de454/f 2023-09-06T20:32:18,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-09-06T20:32:18,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(625): Region bounds: first= last= 2023-09-06T20:32:18,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HRegion(2520): Flush status journal for f63c4b76a967620ba02f0f167d7de454: 2023-09-06T20:32:18,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-test-1694032296496-1694032335819/f/3847e532996147c091bb820ef5a3f90e to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__test-1694032296496__u7fbknv1uq1in1q6f5055b4p2sn57i1bma7v9kkb45e70526ptcibseepq59aip5/f/3847e532996147c091bb820ef5a3f90e 2023-09-06T20:32:18,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__test-1694032296496__u7fbknv1uq1in1q6f5055b4p2sn57i1bma7v9kkb45e70526ptcibseepq59aip5/f/3847e532996147c091bb820ef5a3f90e as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/f/399ab11276b34883846f75d55f1892df_SeqId_4_ 2023-09-06T20:32:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5ba98c41 to 127.0.0.1:53515 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:32:18,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a1f05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:18,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:32:18,461 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45492, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:32:18,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:32:18,467 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41274, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:32:18,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] impl.BackupSystemTable(408): write bulk load descriptor to backup test-1694032296496 with 1 entries 2023-09-06T20:32:18,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] impl.BackupSystemTable(1582): writing raw bulk path hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/f/399ab11276b34883846f75d55f1892df_SeqId_4_ for test-1694032296496 f63c4b76a967620ba02f0f167d7de454 2023-09-06T20:32:18,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] impl.BackupSystemTable(415): written 1 rows for bulk load of test-1694032296496 2023-09-06T20:32:18,490 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:32:18,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5ba98c41 to 127.0.0.1:53515 2023-09-06T20:32:18,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:32:18,492 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__test-1694032296496__u7fbknv1uq1in1q6f5055b4p2sn57i1bma7v9kkb45e70526ptcibseepq59aip5/f/3847e532996147c091bb820ef5a3f90e into f63c4b76a967620ba02f0f167d7de454/f as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/f/399ab11276b34883846f75d55f1892df_SeqId_4_ - updating store file list. 2023-09-06T20:32:18,498 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/f/399ab11276b34883846f75d55f1892df_SeqId_4_ into f63c4b76a967620ba02f0f167d7de454/f 2023-09-06T20:32:18,498 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__test-1694032296496__u7fbknv1uq1in1q6f5055b4p2sn57i1bma7v9kkb45e70526ptcibseepq59aip5/f/3847e532996147c091bb820ef5a3f90e into f63c4b76a967620ba02f0f167d7de454/f (new location: hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/f/399ab11276b34883846f75d55f1892df_SeqId_4_) 2023-09-06T20:32:18,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__test-1694032296496__u7fbknv1uq1in1q6f5055b4p2sn57i1bma7v9kkb45e70526ptcibseepq59aip5/f/3847e532996147c091bb820ef5a3f90e 2023-09-06T20:32:18,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0d91f969 to 127.0.0.1:53515 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:32:18,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d29d806, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:18,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:32:18,508 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45496, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-09-06T20:32:18,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:32:18,513 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41288, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-09-06T20:32:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] impl.BackupSystemTable(388): write bulk load descriptor to backup test-1694032296496 with 1 entries 2023-09-06T20:32:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] impl.BackupSystemTable(1503): writing done bulk path hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/f/399ab11276b34883846f75d55f1892df_SeqId_4_ for test-1694032296496 f63c4b76a967620ba02f0f167d7de454 2023-09-06T20:32:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] impl.BackupSystemTable(394): written 1 rows for bulk load of test-1694032296496 2023-09-06T20:32:18,532 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:32:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0d91f969 to 127.0.0.1:53515 2023-09-06T20:32:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:32:18,534 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:32:18,535 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1bd2beab to 127.0.0.1:53515 2023-09-06T20:32:18,535 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:32:18,535 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-09-06T20:32:18,535 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-09-06T20:32:18,535 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(233): Backup: backup_1694032326314 hdfs://localhost:39327/backupUT/backup_1694032326314/default/test-1694032296496/ 2023-09-06T20:32:18,535 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-09-06T20:32:18,558 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingleOverwrite Thread=604 (was 598) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53515@0x1bd2beab-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53515@0x5ba98c41-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53515@0x5ba98c41-SendThread(127.0.0.1:53515) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:228) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1338) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1261) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-623328790_16 at /127.0.0.1:59908 [Waiting for operation #15] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53515@0x0d91f969-SendThread(127.0.0.1:53515) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1261) Potentially hanging thread: hconnection-0x1f412458-shared-pool-9 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1247140837-172.31.12.81-1694032281151:blk_1073741883_1059, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x77470b9a-shared-pool-24 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-508974552_16 at /127.0.0.1:59842 [Receiving block BP-1247140837-172.31.12.81-1694032281151:blk_1073741882_1058] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-508974552_16 at /127.0.0.1:59844 [Receiving block BP-1247140837-172.31.12.81-1694032281151:blk_1073741883_1059] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53515@0x1bd2beab java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1529) org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1512) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1481) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$490/1571050759.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53515@0x1bd2beab-SendThread(127.0.0.1:53515) sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:345) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1223) Potentially hanging thread: HFileArchiver-3 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53515@0x0d91f969-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-508974552_16 at /127.0.0.1:59896 [Waiting for operation #20] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53515@0x5ba98c41 java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1455) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1485) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$490/1571050759.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53515@0x0d91f969 java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1455) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1485) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$490/1571050759.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x77470b9a-shared-pool-23 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1247140837-172.31.12.81-1694032281151:blk_1073741882_1058, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=694 (was 669) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=225 (was 259), ProcessCount=167 (was 167), AvailableMemoryMB=3527 (was 3702) 2023-09-06T20:32:18,558 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(130): Thread=604 is superior to 500 2023-09-06T20:32:18,574 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingleDNECommand Thread=604, OpenFileDescriptor=694, MaxFileDescriptor=60000, SystemLoadAverage=225, ProcessCount=167, AvailableMemoryMB=3525 2023-09-06T20:32:18,574 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(130): Thread=604 is superior to 500 2023-09-06T20:32:18,574 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] backup.TestFullRestore(280): test restore fails on a single table that does not exist: command-line 2023-09-06T20:32:18,575 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4a8cb598 to 127.0.0.1:53515 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:32:18,582 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@709611b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:18,587 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:32:18,588 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41292, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:32:18,590 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:32:18,591 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45500, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:32:18,606 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:32:18,612 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:32:18,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032338612 (current time:1694032338612). 2023-09-06T20:32:18,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:32:18,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:32:18,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x792ee06e to 127.0.0.1:53515 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:32:18,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@437602d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:18,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:32:18,621 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41308, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:32:18,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x792ee06e to 127.0.0.1:53515 2023-09-06T20:32:18,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:32:18,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:32:18,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=60, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:32:18,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2023-09-06T20:32:18,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-09-06T20:32:18,626 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:32:18,627 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:32:18,629 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:32:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-09-06T20:32:18,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-09-06T20:32:19,037 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:32:19,037 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281}] 2023-09-06T20:32:19,041 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=61, ppid=60, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 2023-09-06T20:32:19,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=61 2023-09-06T20:32:19,192 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:32:19,192 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2819): Flushing cf63a85c53279bc94811b50578f31281 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-09-06T20:32:19,196 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/dd7edbeade334fa7895ff079e0fc3472 is 169, key is trslm:hdfs://localhost:39327/backupUT\x00test-1694032296496/meta:log-roll-map/1694032334442/Put/seqid=0 2023-09-06T20:32:19,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-09-06T20:32:19,601 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/dd7edbeade334fa7895ff079e0fc3472 2023-09-06T20:32:19,609 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/bcfde09dc46641bebf14d756f321d0af is 452, key is session:backup_1694032326314/session:context/1694032334866/Put/seqid=0 2023-09-06T20:32:19,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-09-06T20:32:20,014 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/bcfde09dc46641bebf14d756f321d0af 2023-09-06T20:32:20,020 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/dd7edbeade334fa7895ff079e0fc3472 as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/dd7edbeade334fa7895ff079e0fc3472 2023-09-06T20:32:20,026 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/dd7edbeade334fa7895ff079e0fc3472, entries=4, sequenceid=28, filesize=5.5 K 2023-09-06T20:32:20,027 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/bcfde09dc46641bebf14d756f321d0af as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/bcfde09dc46641bebf14d756f321d0af 2023-09-06T20:32:20,033 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/bcfde09dc46641bebf14d756f321d0af, entries=2, sequenceid=28, filesize=5.4 K 2023-09-06T20:32:20,034 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for cf63a85c53279bc94811b50578f31281 in 842ms, sequenceid=28, compaction requested=true 2023-09-06T20:32:20,034 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2520): Flush status journal for cf63a85c53279bc94811b50578f31281: 2023-09-06T20:32:20,034 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. for snapshot_backup_system completed. 2023-09-06T20:32:20,034 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:32:20,034 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:32:20,034 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/0379b1e1f59e40d2ac2991b2ae6b86ce, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/dd7edbeade334fa7895ff079e0fc3472] hfiles 2023-09-06T20:32:20,034 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/0379b1e1f59e40d2ac2991b2ae6b86ce for snapshot=snapshot_backup_system 2023-09-06T20:32:20,034 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/dd7edbeade334fa7895ff079e0fc3472 for snapshot=snapshot_backup_system 2023-09-06T20:32:20,034 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/4ced064cdbd34bd6b594c76f049fa4ee, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/baca9a30045a4dabad9b3b78a8fd6c86, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/bcfde09dc46641bebf14d756f321d0af] hfiles 2023-09-06T20:32:20,035 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/4ced064cdbd34bd6b594c76f049fa4ee for snapshot=snapshot_backup_system 2023-09-06T20:32:20,035 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/baca9a30045a4dabad9b3b78a8fd6c86 for snapshot=snapshot_backup_system 2023-09-06T20:32:20,035 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/bcfde09dc46641bebf14d756f321d0af for snapshot=snapshot_backup_system 2023-09-06T20:32:20,440 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:32:20,440 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=61 2023-09-06T20:32:20,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=61 2023-09-06T20:32:20,441 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region cf63a85c53279bc94811b50578f31281 2023-09-06T20:32:20,443 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=61, ppid=60, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 2023-09-06T20:32:20,445 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=61, resume processing ppid=60 2023-09-06T20:32:20,445 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:32:20,445 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=61, ppid=60, state=SUCCESS; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 in 1.4060 sec 2023-09-06T20:32:20,446 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:32:20,446 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:32:20,446 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:32:20,447 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:32:20,624 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:32:20,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-09-06T20:32:20,857 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:32:20,865 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:32:20,865 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:32:20,866 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:32:20,866 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2023-09-06T20:32:20,867 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=60, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2430 sec 2023-09-06T20:32:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-09-06T20:32:22,733 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: backup:system, procId: 60 completed 2023-09-06T20:32:22,733 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(115): Backup backup_1694032338582 started at 1694032342733. 2023-09-06T20:32:22,735 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(119): Backup session backup_1694032338582 has been started. 2023-09-06T20:32:22,740 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-09-06T20:32:22,740 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-09-06T20:32:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-09-06T20:32:22,741 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-09-06T20:32:22,741 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-09-06T20:32:22,742 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-09-06T20:32:22,742 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-09-06T20:32:22,742 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-09-06T20:32:22,742 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:32:22,743 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-09-06T20:32:22,743 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:22,744 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:32:22,744 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:32:22,744 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:22,744 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-09-06T20:32:22,744 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-09-06T20:32:22,744 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-09-06T20:32:22,745 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-09-06T20:32:22,745 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:32:22,745 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-09-06T20:32:22,745 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-09-06T20:32:22,745 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-09-06T20:32:22,745 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-09-06T20:32:22,745 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-09-06T20:32:22,746 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-09-06T20:32:22,746 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-09-06T20:32:22,746 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,36961,1694032287907' joining acquired barrier for procedure (rolllog) in zk 2023-09-06T20:32:22,748 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-09-06T20:32:22,748 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:22,748 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:22,748 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:32:22,748 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:32:22,748 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-09-06T20:32:22,748 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-09-06T20:32:22,748 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:32:22,749 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:32:22,749 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:32:22,749 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:22,749 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:32:22,749 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,36961,1694032287907' joining acquired barrier for procedure 'rolllog' on coordinator 2023-09-06T20:32:22,749 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@6b1454e8[Count = 0] remaining members to acquire global barrier 2023-09-06T20:32:22,749 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-09-06T20:32:22,750 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-09-06T20:32:22,751 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-09-06T20:32:22,751 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-09-06T20:32:22,751 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-09-06T20:32:22,751 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-09-06T20:32:22,751 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:22,751 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-09-06T20:32:22,751 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-09-06T20:32:22,751 DEBUG [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:22,751 INFO [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1694032330507 highest: 1694032330507 on jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:22,751 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta:.meta(num 1694032330489) roll requested 2023-09-06T20:32:22,763 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032342751.meta, exclude list is [], retry=0 2023-09-06T20:32:22,767 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK] 2023-09-06T20:32:22,769 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032330489.meta with entries=12, filesize=3.07 KB; new WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032342751.meta 2023-09-06T20:32:22,769 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK]] 2023-09-06T20:32:22,769 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032330489.meta is not closed yet, will try archiving it next time 2023-09-06T20:32:22,769 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36961%2C1694032287907:(num 1694032330507) roll requested 2023-09-06T20:32:22,781 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032342769, exclude list is [], retry=0 2023-09-06T20:32:22,784 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK] 2023-09-06T20:32:22,786 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032330507 with entries=17, filesize=5.25 KB; new WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032342769 2023-09-06T20:32:22,786 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK]] 2023-09-06T20:32:22,786 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032330507 is not closed yet, will try archiving it next time 2023-09-06T20:32:22,791 DEBUG [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-09-06T20:32:22,791 INFO [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1694032342769 on jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:22,801 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-09-06T20:32:22,801 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,36961,1694032287907' in zk 2023-09-06T20:32:22,804 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:22,805 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-09-06T20:32:22,805 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:32:22,805 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-09-06T20:32:22,805 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:22,805 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:32:22,805 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:32:22,806 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:32:22,806 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:32:22,807 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:32:22,807 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:22,808 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:32:22,808 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:32:22,808 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:22,809 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,36961,1694032287907': 2023-09-06T20:32:22,809 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,36961,1694032287907' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-09-06T20:32:22,809 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-09-06T20:32:22,809 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-09-06T20:32:22,809 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-09-06T20:32:22,809 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-09-06T20:32:22,809 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-09-06T20:32:22,811 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-09-06T20:32:22,811 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-09-06T20:32:22,811 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-09-06T20:32:22,811 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-09-06T20:32:22,811 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:32:22,811 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-09-06T20:32:22,811 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:32:22,811 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:32:22,811 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:32:22,811 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:32:22,811 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:32:22,811 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:22,812 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:32:22,812 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:32:22,812 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:32:22,812 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:32:22,813 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:22,813 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:22,813 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:32:22,813 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:32:22,814 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:22,817 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-09-06T20:32:22,817 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36961-0x100eefaa8100001, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-09-06T20:32:22,817 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:32:22,817 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:32:22,817 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:32:22,817 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:22,817 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-09-06T20:32:22,817 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2023-09-06T20:32:22,817 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-09-06T20:32:22,817 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-09-06T20:32:22,818 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-09-06T20:32:22,818 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:22,818 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-09-06T20:32:22,818 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-09-06T20:32:22,818 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:32:22,818 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:37419-0x100eefaa8100000, quorum=127.0.0.1:53515, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-09-06T20:32:22,818 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:32:22,818 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin(2838): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-09-06T20:32:22,818 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin(2846): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-09-06T20:32:22,918 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin(2852): Getting current status of procedure from master... 2023-09-06T20:32:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-09-06T20:32:22,922 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1694032342921_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } 2023-09-06T20:32:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032342922 (current time:1694032342922). 2023-09-06T20:32:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:32:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1694032342921_default_test-1694032296496 VERSION not specified, setting to 2 2023-09-06T20:32:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x55ad4ebc to 127.0.0.1:53515 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:32:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b8ef9f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:32:22,931 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:58568, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:32:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x55ad4ebc to 127.0.0.1:53515 2023-09-06T20:32:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:32:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:32:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=62, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1694032342921_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } 2023-09-06T20:32:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1694032342921_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-09-06T20:32:22,935 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1694032342921_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:32:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-09-06T20:32:22,936 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1694032342921_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:32:22,937 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1694032342921_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:32:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-09-06T20:32:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-09-06T20:32:23,343 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1694032342921_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:32:23,343 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure f63c4b76a967620ba02f0f167d7de454}] 2023-09-06T20:32:23,345 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure f63c4b76a967620ba02f0f167d7de454 2023-09-06T20:32:23,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=63 2023-09-06T20:32:23,497 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454. 2023-09-06T20:32:23,497 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegion(2520): Flush status journal for f63c4b76a967620ba02f0f167d7de454: 2023-09-06T20:32:23,497 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454. for snapshot_1694032342921_default_test-1694032296496 completed. 2023-09-06T20:32:23,497 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(240): Storing 'test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454.' region-info for snapshot=snapshot_1694032342921_default_test-1694032296496 2023-09-06T20:32:23,497 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:32:23,498 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/f/399ab11276b34883846f75d55f1892df_SeqId_4_] hfiles 2023-09-06T20:32:23,498 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/f/399ab11276b34883846f75d55f1892df_SeqId_4_ for snapshot=snapshot_1694032342921_default_test-1694032296496 2023-09-06T20:32:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-09-06T20:32:23,904 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454. 2023-09-06T20:32:23,904 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=63 2023-09-06T20:32:23,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=63 2023-09-06T20:32:23,904 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1694032342921_default_test-1694032296496 on region f63c4b76a967620ba02f0f167d7de454 2023-09-06T20:32:23,906 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure f63c4b76a967620ba02f0f167d7de454 2023-09-06T20:32:23,908 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=63, resume processing ppid=62 2023-09-06T20:32:23,908 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1694032342921_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:32:23,908 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=63, ppid=62, state=SUCCESS; SnapshotRegionProcedure f63c4b76a967620ba02f0f167d7de454 in 563 msec 2023-09-06T20:32:23,908 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1694032342921_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:32:23,909 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1694032342921_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:32:23,909 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1694032342921_default_test-1694032296496 2023-09-06T20:32:23,911 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032342921_default_test-1694032296496 2023-09-06T20:32:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-09-06T20:32:24,324 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1694032342921_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:32:24,330 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1694032342921_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:32:24,331 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032342921_default_test-1694032296496 to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032342921_default_test-1694032296496 2023-09-06T20:32:24,333 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1694032342921_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:32:24,333 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1694032342921_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-09-06T20:32:24,334 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=62, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1694032342921_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } in 1.4010 sec 2023-09-06T20:32:25,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-09-06T20:32:25,041 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: default:test-1694032296496, procId: 62 completed 2023-09-06T20:32:25,041 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1694032338582 2023-09-06T20:32:25,041 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-09-06T20:32:25,041 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-09-06T20:32:25,042 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1694032338582_test-1694032296496 2023-09-06T20:32:25,042 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1694032342921_default_test-1694032296496 to hdfs://localhost:39327/backupUT/backup_1694032338582/default/test-1694032296496/ 2023-09-06T20:32:25,042 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-09-06T20:32:25,065 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:32:25,065 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37419,1694032284546_-978852072_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:39327/backupUT/backup_1694032338582/default/test-1694032296496, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:39327/backupUT/backup_1694032338582/default/test-1694032296496/.hbase-snapshot/.tmp/snapshot_1694032342921_default_test-1694032296496 2023-09-06T20:32:25,066 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1, snapshotDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032342921_default_test-1694032296496. 2023-09-06T20:32:25,074 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032342921_default_test-1694032296496 to hdfs://localhost:39327/backupUT/backup_1694032338582/default/test-1694032296496/.hbase-snapshot/.tmp/snapshot_1694032342921_default_test-1694032296496 2023-09-06T20:32:25,489 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:25,489 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:25,490 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:25,490 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:26,458 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop-7187948410819950858.jar 2023-09-06T20:32:26,458 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:26,458 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:26,459 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:26,459 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:26,460 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:26,460 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:26,460 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:26,461 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:26,461 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:26,462 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:32:26,463 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:32:26,464 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:32:26,464 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:32:26,465 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:32:26,465 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:32:26,466 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:32:26,466 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:32:26,467 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:32:26,467 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:32:26,468 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:32:26,469 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:32:26,469 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:32:26,470 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:32:26,471 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:32:26,471 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:32:26,472 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:32:26,473 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:32:26,473 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:32:26,505 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-09-06T20:32:26,522 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1694032342921_default_test-1694032296496' hfile list 2023-09-06T20:32:26,524 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-09-06T20:32:26,597 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032346561/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:32:26,597 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032346561/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:32:26,653 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-09-06T20:32:26,678 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/f/399ab11276b34883846f75d55f1892df_SeqId_4_ output=hdfs://localhost:39327/backupUT/backup_1694032338582/default/test-1694032296496/archive/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/f/399ab11276b34883846f75d55f1892df_SeqId_4_ 2023-09-06T20:32:26,678 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 8.320M/sec 2023-09-06T20:32:27,633 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-09-06T20:32:27,634 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-09-06T20:32:27,641 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1694032342921_default_test-1694032296496 2023-09-06T20:32:27,641 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1694032342921_default_test-1694032296496 finished. 2023-09-06T20:32:27,641 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1014): test-1694032296496 2023-09-06T20:32:27,648 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1694032338582 2023-09-06T20:32:27,648 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-09-06T20:32:28,053 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:39327/backupUT/backup_1694032338582/.backup.manifest 2023-09-06T20:32:28,053 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(358): Backup backup_1694032338582 finished: type=FULL,tablelist=test-1694032296496,targetRootDir=hdfs://localhost:39327/backupUT,startts=1694032342733,completets=1694032347647,bytescopied=0 2023-09-06T20:32:28,053 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-09-06T20:32:28,053 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1694032342921_default_test-1694032296496 2023-09-06T20:32:28,054 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1694032342921_default_test-1694032296496" 2023-09-06T20:32:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1694032342921_default_test-1694032296496 2023-09-06T20:32:28,057 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1694032342921_default_test-1694032296496 for backup backup_1694032338582 succeeded. 2023-09-06T20:32:28,059 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:32:28,063 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:32:28,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:32:28,066 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:32:28,068 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:32:28,074 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(376): Backup backup_1694032338582 completed. 2023-09-06T20:32:28,074 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-09-06T20:32:28,074 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4a8cb598 to 127.0.0.1:53515 2023-09-06T20:32:28,074 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:32:28,086 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] backup.TestFullRestore(285): backup complete You have to specify either set name or table list to restore Usage: hbase restore [options] backup_path Path to a backup destination root backup_id Backup image ID to restore table(s) Comma-separated list of tables to restore Options: -c Check restore sequence and dependencies only (does not execute the command) -d Enable debug loggings -h,--help Show usage -m A comma separated list of target tables. If specified, each table in must have a mapping -o Overwrite data if any of the restore target tables exists -q Yarn queue name to run backup restore command on -s Backup set to restore, mutually exclusive with -t (table list) -t Table name list, comma-separated. To enable backup, in hbase-site.xml, set: hbase.backup.enable=true hbase.master.logcleaner.plugins=YOUR_PLUGINS,org.apache.hadoop.hbase.backup.master.BackupLogCleaner hbase.procedure.master.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager hbase.procedure.regionserver.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager hbase.coprocessor.region.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.BackupObserver and restart the cluster For more information please see http://hbase.apache.org/book.html#backuprestore 2023-09-06T20:32:28,116 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingleDNECommand Thread=596 (was 604), OpenFileDescriptor=685 (was 694), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=199 (was 225), ProcessCount=167 (was 167), AvailableMemoryMB=3412 (was 3525) 2023-09-06T20:32:28,116 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(130): Thread=596 is superior to 500 2023-09-06T20:32:28,131 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreCheckCommand Thread=596, OpenFileDescriptor=685, MaxFileDescriptor=60000, SystemLoadAverage=199, ProcessCount=167, AvailableMemoryMB=3411 2023-09-06T20:32:28,131 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(130): Thread=596 is superior to 500 2023-09-06T20:32:28,131 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] backup.TestFullRestore(97): test full restore on a single table: command-line, check only 2023-09-06T20:32:28,136 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528d2d7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:28,140 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:32:28,142 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:58582, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:32:28,144 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:32:28,145 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:47654, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:32:28,156 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:32:28,161 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:32:28,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032348161 (current time:1694032348161). 2023-09-06T20:32:28,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:32:28,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:32:28,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7435e201, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:28,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:32:28,170 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:58592, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:32:28,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:32:28,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:32:28,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=64, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:32:28,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-09-06T20:32:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-09-06T20:32:28,175 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:32:28,176 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:32:28,177 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:32:28,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-09-06T20:32:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-09-06T20:32:28,584 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:32:28,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281}] 2023-09-06T20:32:28,586 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 2023-09-06T20:32:28,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=65 2023-09-06T20:32:28,738 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:32:28,738 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2819): Flushing cf63a85c53279bc94811b50578f31281 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-09-06T20:32:28,742 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/78eec26cfa0242e48353b19a4a00b7d5 is 169, key is trslm:hdfs://localhost:39327/backupUT\x00test-1694032296496/meta:log-roll-map/1694032347643/Put/seqid=0 2023-09-06T20:32:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-09-06T20:32:29,147 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/78eec26cfa0242e48353b19a4a00b7d5 2023-09-06T20:32:29,156 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/2dc68dbd9f3749a1a2d9d7333fa4771d is 452, key is session:backup_1694032338582/session:context/1694032348067/Put/seqid=0 2023-09-06T20:32:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-09-06T20:32:29,560 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/2dc68dbd9f3749a1a2d9d7333fa4771d 2023-09-06T20:32:29,567 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/78eec26cfa0242e48353b19a4a00b7d5 as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/78eec26cfa0242e48353b19a4a00b7d5 2023-09-06T20:32:29,572 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/78eec26cfa0242e48353b19a4a00b7d5, entries=4, sequenceid=39, filesize=5.5 K 2023-09-06T20:32:29,573 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/2dc68dbd9f3749a1a2d9d7333fa4771d as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/2dc68dbd9f3749a1a2d9d7333fa4771d 2023-09-06T20:32:29,578 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/2dc68dbd9f3749a1a2d9d7333fa4771d, entries=2, sequenceid=39, filesize=5.4 K 2023-09-06T20:32:29,579 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for cf63a85c53279bc94811b50578f31281 in 840ms, sequenceid=39, compaction requested=true 2023-09-06T20:32:29,579 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2520): Flush status journal for cf63a85c53279bc94811b50578f31281: 2023-09-06T20:32:29,579 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. for snapshot_backup_system completed. 2023-09-06T20:32:29,579 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:32:29,579 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:32:29,579 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/0379b1e1f59e40d2ac2991b2ae6b86ce, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/dd7edbeade334fa7895ff079e0fc3472, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/78eec26cfa0242e48353b19a4a00b7d5] hfiles 2023-09-06T20:32:29,579 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/0379b1e1f59e40d2ac2991b2ae6b86ce for snapshot=snapshot_backup_system 2023-09-06T20:32:29,579 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/dd7edbeade334fa7895ff079e0fc3472 for snapshot=snapshot_backup_system 2023-09-06T20:32:29,579 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/78eec26cfa0242e48353b19a4a00b7d5 for snapshot=snapshot_backup_system 2023-09-06T20:32:29,579 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/4ced064cdbd34bd6b594c76f049fa4ee, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/baca9a30045a4dabad9b3b78a8fd6c86, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/bcfde09dc46641bebf14d756f321d0af, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/2dc68dbd9f3749a1a2d9d7333fa4771d] hfiles 2023-09-06T20:32:29,579 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/4ced064cdbd34bd6b594c76f049fa4ee for snapshot=snapshot_backup_system 2023-09-06T20:32:29,579 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/baca9a30045a4dabad9b3b78a8fd6c86 for snapshot=snapshot_backup_system 2023-09-06T20:32:29,579 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/bcfde09dc46641bebf14d756f321d0af for snapshot=snapshot_backup_system 2023-09-06T20:32:29,579 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/2dc68dbd9f3749a1a2d9d7333fa4771d for snapshot=snapshot_backup_system 2023-09-06T20:32:29,901 DEBUG [regionserver/jenkins-hbase3:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.07692307692307693, tune throughput to 53.85 MB/second 2023-09-06T20:32:29,985 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:32:29,985 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=65 2023-09-06T20:32:29,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=65 2023-09-06T20:32:29,985 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region cf63a85c53279bc94811b50578f31281 2023-09-06T20:32:29,986 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 2023-09-06T20:32:29,989 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=65, resume processing ppid=64 2023-09-06T20:32:29,989 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:32:29,989 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=65, ppid=64, state=SUCCESS; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 in 1.4020 sec 2023-09-06T20:32:29,989 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:32:29,990 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:32:29,990 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:32:29,991 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:32:30,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-09-06T20:32:30,404 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:32:30,413 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:32:30,414 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:32:30,415 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:32:30,415 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-09-06T20:32:30,416 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=64, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2440 sec 2023-09-06T20:32:31,135 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 72ae4de93b6a56d1410c6ff0eb3e6243 changed from -1.0 to 0.0, refreshing cache 2023-09-06T20:32:32,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-09-06T20:32:32,281 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(115): Backup backup_1694032348136 started at 1694032352280. 2023-09-06T20:32:32,282 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(119): Backup session backup_1694032348136 has been started. 2023-09-06T20:32:32,288 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-09-06T20:32:32,288 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-09-06T20:32:32,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-09-06T20:32:32,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-09-06T20:32:32,289 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-09-06T20:32:32,289 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-09-06T20:32:32,290 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-09-06T20:32:32,290 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-09-06T20:32:32,291 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:32:32,292 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:32,292 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:32:32,292 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:32:32,292 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-09-06T20:32:32,292 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-09-06T20:32:32,293 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-09-06T20:32:32,293 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:32:32,293 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-09-06T20:32:32,293 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-09-06T20:32:32,293 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-09-06T20:32:32,293 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-09-06T20:32:32,293 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-09-06T20:32:32,293 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-09-06T20:32:32,294 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-09-06T20:32:32,294 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-09-06T20:32:32,294 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,36961,1694032287907' joining acquired barrier for procedure (rolllog) in zk 2023-09-06T20:32:32,297 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:32,297 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-09-06T20:32:32,297 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:32:32,297 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:32:32,297 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-09-06T20:32:32,297 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:32:32,297 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:32:32,298 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:32:32,298 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:32,298 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:32:32,299 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,36961,1694032287907' joining acquired barrier for procedure 'rolllog' on coordinator 2023-09-06T20:32:32,299 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@53843285[Count = 0] remaining members to acquire global barrier 2023-09-06T20:32:32,299 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-09-06T20:32:32,299 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-09-06T20:32:32,300 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-09-06T20:32:32,300 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-09-06T20:32:32,300 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-09-06T20:32:32,300 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-09-06T20:32:32,300 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-09-06T20:32:32,300 DEBUG [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:32,301 INFO [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1694032342769 highest: 1694032342769 on jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:32,301 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta:.meta(num 1694032342751) roll requested 2023-09-06T20:32:32,312 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032352301.meta, exclude list is [], retry=0 2023-09-06T20:32:32,317 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK] 2023-09-06T20:32:32,318 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032342751.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032352301.meta 2023-09-06T20:32:32,318 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK]] 2023-09-06T20:32:32,318 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032342751.meta is not closed yet, will try archiving it next time 2023-09-06T20:32:32,318 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36961%2C1694032287907:(num 1694032342769) roll requested 2023-09-06T20:32:32,322 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032342751.meta to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/oldWALs/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032342751.meta 2023-09-06T20:32:32,331 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032352318, exclude list is [], retry=0 2023-09-06T20:32:32,334 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK] 2023-09-06T20:32:32,335 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032342769 with entries=10, filesize=2.31 KB; new WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032352318 2023-09-06T20:32:32,335 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK]] 2023-09-06T20:32:32,336 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032342769 is not closed yet, will try archiving it next time 2023-09-06T20:32:32,341 DEBUG [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-09-06T20:32:32,341 INFO [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1694032352318 on jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:32,350 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-09-06T20:32:32,350 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,36961,1694032287907' in zk 2023-09-06T20:32:32,353 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-09-06T20:32:32,353 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:32:32,353 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:32,353 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-09-06T20:32:32,353 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:32:32,354 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:32:32,354 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:32:32,355 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:32:32,355 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:32:32,355 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:32,355 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:32:32,356 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:32:32,356 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:32,356 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,36961,1694032287907': 2023-09-06T20:32:32,356 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,36961,1694032287907' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-09-06T20:32:32,356 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-09-06T20:32:32,357 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-09-06T20:32:32,357 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-09-06T20:32:32,357 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-09-06T20:32:32,357 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-09-06T20:32:32,358 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-09-06T20:32:32,358 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:32:32,358 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:32:32,358 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-09-06T20:32:32,358 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:32:32,359 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:32:32,359 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:32:32,359 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:32:32,359 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:32:32,359 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:32:32,359 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:32:32,360 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:32:32,360 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:32,360 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:32:32,361 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:32:32,361 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:32,365 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:32:32,365 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:32:32,365 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:32:32,365 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-09-06T20:32:32,365 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-09-06T20:32:32,366 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:32:32,366 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:32:32,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-09-06T20:32:32,470 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1694032352470_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } 2023-09-06T20:32:32,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032352470 (current time:1694032352470). 2023-09-06T20:32:32,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:32:32,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1694032352470_default_test-1694032296496 VERSION not specified, setting to 2 2023-09-06T20:32:32,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ae587dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:32,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:32:32,480 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46740, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:32:32,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:32:32,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:32:32,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=66, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1694032352470_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } 2023-09-06T20:32:32,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1694032352470_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 }, snapshot procedure id = 66 2023-09-06T20:32:32,484 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1694032352470_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:32:32,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-09-06T20:32:32,485 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1694032352470_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:32:32,486 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1694032352470_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:32:32,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-09-06T20:32:32,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-09-06T20:32:32,892 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1694032352470_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:32:32,892 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure f63c4b76a967620ba02f0f167d7de454}] 2023-09-06T20:32:32,894 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure f63c4b76a967620ba02f0f167d7de454 2023-09-06T20:32:33,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=67 2023-09-06T20:32:33,046 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454. 2023-09-06T20:32:33,046 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HRegion(2520): Flush status journal for f63c4b76a967620ba02f0f167d7de454: 2023-09-06T20:32:33,046 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454. for snapshot_1694032352470_default_test-1694032296496 completed. 2023-09-06T20:32:33,046 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(240): Storing 'test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454.' region-info for snapshot=snapshot_1694032352470_default_test-1694032296496 2023-09-06T20:32:33,046 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:32:33,047 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/f/399ab11276b34883846f75d55f1892df_SeqId_4_] hfiles 2023-09-06T20:32:33,047 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/f/399ab11276b34883846f75d55f1892df_SeqId_4_ for snapshot=snapshot_1694032352470_default_test-1694032296496 2023-09-06T20:32:33,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-09-06T20:32:33,454 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454. 2023-09-06T20:32:33,454 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=67 2023-09-06T20:32:33,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=67 2023-09-06T20:32:33,455 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1694032352470_default_test-1694032296496 on region f63c4b76a967620ba02f0f167d7de454 2023-09-06T20:32:33,456 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure f63c4b76a967620ba02f0f167d7de454 2023-09-06T20:32:33,458 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=67, resume processing ppid=66 2023-09-06T20:32:33,458 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=67, ppid=66, state=SUCCESS; SnapshotRegionProcedure f63c4b76a967620ba02f0f167d7de454 in 564 msec 2023-09-06T20:32:33,458 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1694032352470_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:32:33,459 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1694032352470_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:32:33,459 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1694032352470_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:32:33,459 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1694032352470_default_test-1694032296496 2023-09-06T20:32:33,460 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032352470_default_test-1694032296496 2023-09-06T20:32:33,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-09-06T20:32:33,870 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1694032352470_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:32:33,876 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1694032352470_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:32:33,876 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032352470_default_test-1694032296496 to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032352470_default_test-1694032296496 2023-09-06T20:32:33,877 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1694032352470_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:32:33,878 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1694032352470_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 }, snapshot procedure id = 66 2023-09-06T20:32:33,879 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=66, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1694032352470_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } in 1.3960 sec 2023-09-06T20:32:34,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-09-06T20:32:34,591 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1694032348136 2023-09-06T20:32:34,591 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-09-06T20:32:34,591 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-09-06T20:32:34,591 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1694032348136_test-1694032296496 2023-09-06T20:32:34,592 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1694032352470_default_test-1694032296496 to hdfs://localhost:39327/backupUT/backup_1694032348136/default/test-1694032296496/ 2023-09-06T20:32:34,592 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-09-06T20:32:34,625 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:32:34,625 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37419,1694032284546_92613670_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:39327/backupUT/backup_1694032348136/default/test-1694032296496, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:39327/backupUT/backup_1694032348136/default/test-1694032296496/.hbase-snapshot/.tmp/snapshot_1694032352470_default_test-1694032296496 2023-09-06T20:32:34,625 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1, snapshotDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032352470_default_test-1694032296496. 2023-09-06T20:32:34,633 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032352470_default_test-1694032296496 to hdfs://localhost:39327/backupUT/backup_1694032348136/default/test-1694032296496/.hbase-snapshot/.tmp/snapshot_1694032352470_default_test-1694032296496 2023-09-06T20:32:35,048 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:35,049 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:35,049 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:35,050 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:36,040 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop-4100865706859847211.jar 2023-09-06T20:32:36,040 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:36,040 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:36,041 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:36,042 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:36,042 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:36,043 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:36,043 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:36,043 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:36,044 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:36,044 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:32:36,045 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:32:36,045 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:32:36,046 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:32:36,046 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:32:36,047 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:32:36,047 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:32:36,047 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:32:36,048 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:32:36,048 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:32:36,048 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:32:36,049 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:32:36,049 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:32:36,050 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:32:36,051 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:32:36,051 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:32:36,051 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:32:36,052 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:32:36,052 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:32:36,095 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1694032352470_default_test-1694032296496' hfile list 2023-09-06T20:32:36,098 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-09-06T20:32:36,164 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032356132/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:32:36,164 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032356132/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:32:36,217 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-09-06T20:32:36,242 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/f/399ab11276b34883846f75d55f1892df_SeqId_4_ output=hdfs://localhost:39327/backupUT/backup_1694032348136/default/test-1694032296496/archive/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/f/399ab11276b34883846f75d55f1892df_SeqId_4_ 2023-09-06T20:32:36,242 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 8.320M/sec 2023-09-06T20:32:37,199 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-09-06T20:32:37,200 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-09-06T20:32:37,207 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1694032352470_default_test-1694032296496 2023-09-06T20:32:37,207 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1694032352470_default_test-1694032296496 finished. 2023-09-06T20:32:37,207 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1014): test-1694032296496 2023-09-06T20:32:37,213 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1694032348136 2023-09-06T20:32:37,213 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-09-06T20:32:37,619 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:39327/backupUT/backup_1694032348136/.backup.manifest 2023-09-06T20:32:37,619 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(358): Backup backup_1694032348136 finished: type=FULL,tablelist=test-1694032296496,targetRootDir=hdfs://localhost:39327/backupUT,startts=1694032352280,completets=1694032357213,bytescopied=0 2023-09-06T20:32:37,619 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-09-06T20:32:37,619 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1694032352470_default_test-1694032296496 2023-09-06T20:32:37,620 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1694032352470_default_test-1694032296496" 2023-09-06T20:32:37,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1694032352470_default_test-1694032296496 2023-09-06T20:32:37,624 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1694032352470_default_test-1694032296496 for backup backup_1694032348136 succeeded. 2023-09-06T20:32:37,625 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:32:37,630 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:32:37,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:32:37,633 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:32:37,634 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:32:37,640 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(376): Backup backup_1694032348136 completed. 2023-09-06T20:32:37,640 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:32:37,641 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] backup.TestFullRestore(101): backup complete 2023-09-06T20:32:37,652 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] backup.RestoreDriver(108): Found -check option in restore command, will check and verify the dependencies 2023-09-06T20:32:37,658 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39349ad5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:37,660 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:39327/backupUT/backup_1694032348136/.backup.manifest 2023-09-06T20:32:37,662 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032348136/.backup.manifest 2023-09-06T20:32:37,662 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] util.BackupUtils(677): Dependent image(s) from old to new: 2023-09-06T20:32:37,663 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] util.BackupUtils(686): Backup image: backup_1694032348136 for 'test-1694032296496' is available 2023-09-06T20:32:37,663 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupAdminImpl(511): Checking backup images: OK 2023-09-06T20:32:37,663 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:32:37,681 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreCheckCommand Thread=595 (was 596), OpenFileDescriptor=677 (was 685), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=169 (was 199), ProcessCount=167 (was 167), AvailableMemoryMB=3324 (was 3411) 2023-09-06T20:32:37,681 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(130): Thread=595 is superior to 500 2023-09-06T20:32:37,696 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingle Thread=595, OpenFileDescriptor=677, MaxFileDescriptor=60000, SystemLoadAverage=169, ProcessCount=167, AvailableMemoryMB=3324 2023-09-06T20:32:37,696 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(130): Thread=595 is superior to 500 2023-09-06T20:32:37,696 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] backup.TestFullRestore(55): test full restore on a single table empty table 2023-09-06T20:32:37,701 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@354928fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:37,704 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:32:37,706 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46746, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:32:37,708 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:32:37,709 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:58746, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:32:37,724 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:32:37,729 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:32:37,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032357729 (current time:1694032357729). 2023-09-06T20:32:37,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:32:37,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:32:37,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f4f890e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:37,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:32:37,737 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46754, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:32:37,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:32:37,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:32:37,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=68, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:32:37,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 68 2023-09-06T20:32:37,741 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:32:37,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-09-06T20:32:37,741 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:32:37,743 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:32:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-09-06T20:32:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-09-06T20:32:38,149 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:32:38,149 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281}] 2023-09-06T20:32:38,151 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=69, ppid=68, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 2023-09-06T20:32:38,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=69 2023-09-06T20:32:38,302 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:32:38,303 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HRegion(2819): Flushing cf63a85c53279bc94811b50578f31281 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-09-06T20:32:38,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/faffbcb72f8649bcad705936a3105fd4 is 169, key is trslm:hdfs://localhost:39327/backupUT\x00test-1694032296496/meta:log-roll-map/1694032357210/Put/seqid=0 2023-09-06T20:32:38,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-09-06T20:32:38,735 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/faffbcb72f8649bcad705936a3105fd4 2023-09-06T20:32:38,744 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/6da826e7b47e41daa41e268fb4e951b2 is 452, key is session:backup_1694032348136/session:context/1694032357633/Put/seqid=0 2023-09-06T20:32:38,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-09-06T20:32:39,149 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/6da826e7b47e41daa41e268fb4e951b2 2023-09-06T20:32:39,155 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/faffbcb72f8649bcad705936a3105fd4 as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/faffbcb72f8649bcad705936a3105fd4 2023-09-06T20:32:39,160 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/faffbcb72f8649bcad705936a3105fd4, entries=4, sequenceid=50, filesize=5.5 K 2023-09-06T20:32:39,161 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/6da826e7b47e41daa41e268fb4e951b2 as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/6da826e7b47e41daa41e268fb4e951b2 2023-09-06T20:32:39,166 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/6da826e7b47e41daa41e268fb4e951b2, entries=2, sequenceid=50, filesize=5.4 K 2023-09-06T20:32:39,167 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for cf63a85c53279bc94811b50578f31281 in 864ms, sequenceid=50, compaction requested=true 2023-09-06T20:32:39,167 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HRegion(2520): Flush status journal for cf63a85c53279bc94811b50578f31281: 2023-09-06T20:32:39,167 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. for snapshot_backup_system completed. 2023-09-06T20:32:39,167 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:32:39,167 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:32:39,167 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/0379b1e1f59e40d2ac2991b2ae6b86ce, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/dd7edbeade334fa7895ff079e0fc3472, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/78eec26cfa0242e48353b19a4a00b7d5, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/faffbcb72f8649bcad705936a3105fd4] hfiles 2023-09-06T20:32:39,167 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/0379b1e1f59e40d2ac2991b2ae6b86ce for snapshot=snapshot_backup_system 2023-09-06T20:32:39,168 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/dd7edbeade334fa7895ff079e0fc3472 for snapshot=snapshot_backup_system 2023-09-06T20:32:39,168 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/78eec26cfa0242e48353b19a4a00b7d5 for snapshot=snapshot_backup_system 2023-09-06T20:32:39,168 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/faffbcb72f8649bcad705936a3105fd4 for snapshot=snapshot_backup_system 2023-09-06T20:32:39,168 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/4ced064cdbd34bd6b594c76f049fa4ee, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/baca9a30045a4dabad9b3b78a8fd6c86, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/bcfde09dc46641bebf14d756f321d0af, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/2dc68dbd9f3749a1a2d9d7333fa4771d, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/6da826e7b47e41daa41e268fb4e951b2] hfiles 2023-09-06T20:32:39,168 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/4ced064cdbd34bd6b594c76f049fa4ee for snapshot=snapshot_backup_system 2023-09-06T20:32:39,168 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/baca9a30045a4dabad9b3b78a8fd6c86 for snapshot=snapshot_backup_system 2023-09-06T20:32:39,168 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/bcfde09dc46641bebf14d756f321d0af for snapshot=snapshot_backup_system 2023-09-06T20:32:39,168 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/2dc68dbd9f3749a1a2d9d7333fa4771d for snapshot=snapshot_backup_system 2023-09-06T20:32:39,168 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/6da826e7b47e41daa41e268fb4e951b2 for snapshot=snapshot_backup_system 2023-09-06T20:32:39,573 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:32:39,573 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=69 2023-09-06T20:32:39,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=69 2023-09-06T20:32:39,574 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region cf63a85c53279bc94811b50578f31281 2023-09-06T20:32:39,575 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=69, ppid=68, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 2023-09-06T20:32:39,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=69, resume processing ppid=68 2023-09-06T20:32:39,577 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:32:39,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=69, ppid=68, state=SUCCESS; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 in 1.4270 sec 2023-09-06T20:32:39,577 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:32:39,578 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:32:39,578 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:32:39,579 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:32:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-09-06T20:32:39,989 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:32:39,999 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:32:39,999 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:32:40,000 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:32:40,000 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 68 2023-09-06T20:32:40,001 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=68, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2610 sec 2023-09-06T20:32:41,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-09-06T20:32:41,849 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(115): Backup backup_1694032357701 started at 1694032361849. 2023-09-06T20:32:41,851 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(119): Backup session backup_1694032357701 has been started. 2023-09-06T20:32:41,855 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-09-06T20:32:41,856 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-09-06T20:32:41,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-09-06T20:32:41,856 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-09-06T20:32:41,856 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-09-06T20:32:41,857 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-09-06T20:32:41,857 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-09-06T20:32:41,858 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:32:41,859 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:41,859 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:32:41,859 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:32:41,859 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-09-06T20:32:41,859 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-09-06T20:32:41,860 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-09-06T20:32:41,860 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:32:41,860 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-09-06T20:32:41,860 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-09-06T20:32:41,860 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-09-06T20:32:41,860 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-09-06T20:32:41,861 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-09-06T20:32:41,861 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-09-06T20:32:41,861 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-09-06T20:32:41,861 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,36961,1694032287907' joining acquired barrier for procedure (rolllog) in zk 2023-09-06T20:32:41,863 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-09-06T20:32:41,863 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:41,864 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:32:41,864 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:32:41,864 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-09-06T20:32:41,864 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:32:41,864 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:32:41,864 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:32:41,865 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:41,865 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:32:41,865 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,36961,1694032287907' joining acquired barrier for procedure 'rolllog' on coordinator 2023-09-06T20:32:41,865 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@5f68dbef[Count = 0] remaining members to acquire global barrier 2023-09-06T20:32:41,865 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-09-06T20:32:41,865 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-09-06T20:32:41,866 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-09-06T20:32:41,866 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-09-06T20:32:41,866 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-09-06T20:32:41,867 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-09-06T20:32:41,867 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-09-06T20:32:41,867 DEBUG [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:41,867 INFO [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1694032352318 highest: 1694032352318 on jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:41,867 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta:.meta(num 1694032352301) roll requested 2023-09-06T20:32:41,878 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032361867.meta, exclude list is [], retry=0 2023-09-06T20:32:41,882 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK] 2023-09-06T20:32:41,884 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032352301.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032361867.meta 2023-09-06T20:32:41,884 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK]] 2023-09-06T20:32:41,884 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032352301.meta is not closed yet, will try archiving it next time 2023-09-06T20:32:41,884 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36961%2C1694032287907:(num 1694032352318) roll requested 2023-09-06T20:32:41,886 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032352301.meta to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/oldWALs/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032352301.meta 2023-09-06T20:32:41,897 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032361884, exclude list is [], retry=0 2023-09-06T20:32:41,900 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK] 2023-09-06T20:32:41,901 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032352318 with entries=10, filesize=2.31 KB; new WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032361884 2023-09-06T20:32:41,901 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK]] 2023-09-06T20:32:41,901 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032352318 is not closed yet, will try archiving it next time 2023-09-06T20:32:41,902 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032342769 to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/oldWALs/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032342769 2023-09-06T20:32:41,907 DEBUG [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-09-06T20:32:41,907 INFO [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1694032361884 on jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:41,915 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-09-06T20:32:41,915 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,36961,1694032287907' in zk 2023-09-06T20:32:41,918 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-09-06T20:32:41,918 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:41,918 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:32:41,918 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:32:41,918 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:32:41,918 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-09-06T20:32:41,919 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:32:41,920 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:32:41,920 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:32:41,920 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:41,921 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:32:41,921 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:32:41,921 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:41,922 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,36961,1694032287907': 2023-09-06T20:32:41,922 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,36961,1694032287907' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-09-06T20:32:41,922 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-09-06T20:32:41,922 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-09-06T20:32:41,922 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-09-06T20:32:41,922 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-09-06T20:32:41,922 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-09-06T20:32:41,923 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-09-06T20:32:41,923 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:32:41,924 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-09-06T20:32:41,924 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:32:41,924 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:32:41,924 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:32:41,924 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:32:41,924 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:32:41,924 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:32:41,925 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:32:41,925 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:32:41,925 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:32:41,926 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:41,926 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:32:41,926 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:32:41,927 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:41,932 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:32:41,932 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:32:41,932 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:32:41,932 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-09-06T20:32:41,932 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-09-06T20:32:41,933 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:32:41,933 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:32:42,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-09-06T20:32:42,037 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1694032362036_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } 2023-09-06T20:32:42,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032362037 (current time:1694032362037). 2023-09-06T20:32:42,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:32:42,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1694032362036_default_test-1694032296496 VERSION not specified, setting to 2 2023-09-06T20:32:42,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@324a11b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:42,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:32:42,046 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45548, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:32:42,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:32:42,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:32:42,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=70, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1694032362036_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } 2023-09-06T20:32:42,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1694032362036_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 }, snapshot procedure id = 70 2023-09-06T20:32:42,049 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1694032362036_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:32:42,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=70 2023-09-06T20:32:42,050 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1694032362036_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:32:42,052 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1694032362036_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:32:42,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=70 2023-09-06T20:32:42,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=70 2023-09-06T20:32:42,458 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1694032362036_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:32:42,458 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE; SnapshotRegionProcedure f63c4b76a967620ba02f0f167d7de454}] 2023-09-06T20:32:42,460 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=71, ppid=70, state=RUNNABLE; SnapshotRegionProcedure f63c4b76a967620ba02f0f167d7de454 2023-09-06T20:32:42,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=71 2023-09-06T20:32:42,611 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454. 2023-09-06T20:32:42,611 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] regionserver.HRegion(2520): Flush status journal for f63c4b76a967620ba02f0f167d7de454: 2023-09-06T20:32:42,611 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454. for snapshot_1694032362036_default_test-1694032296496 completed. 2023-09-06T20:32:42,612 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] snapshot.SnapshotManifest(240): Storing 'test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454.' region-info for snapshot=snapshot_1694032362036_default_test-1694032296496 2023-09-06T20:32:42,612 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:32:42,612 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/f/399ab11276b34883846f75d55f1892df_SeqId_4_] hfiles 2023-09-06T20:32:42,612 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/f/399ab11276b34883846f75d55f1892df_SeqId_4_ for snapshot=snapshot_1694032362036_default_test-1694032296496 2023-09-06T20:32:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=70 2023-09-06T20:32:43,017 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454. 2023-09-06T20:32:43,017 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=71 2023-09-06T20:32:43,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=71 2023-09-06T20:32:43,018 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1694032362036_default_test-1694032296496 on region f63c4b76a967620ba02f0f167d7de454 2023-09-06T20:32:43,019 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=71, ppid=70, state=RUNNABLE; SnapshotRegionProcedure f63c4b76a967620ba02f0f167d7de454 2023-09-06T20:32:43,021 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=71, resume processing ppid=70 2023-09-06T20:32:43,021 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=71, ppid=70, state=SUCCESS; SnapshotRegionProcedure f63c4b76a967620ba02f0f167d7de454 in 562 msec 2023-09-06T20:32:43,021 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1694032362036_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:32:43,022 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1694032362036_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:32:43,022 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1694032362036_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:32:43,022 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1694032362036_default_test-1694032296496 2023-09-06T20:32:43,023 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032362036_default_test-1694032296496 2023-09-06T20:32:43,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=70 2023-09-06T20:32:43,432 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1694032362036_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:32:43,438 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1694032362036_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:32:43,438 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032362036_default_test-1694032296496 to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032362036_default_test-1694032296496 2023-09-06T20:32:43,439 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1694032362036_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:32:43,439 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1694032362036_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 }, snapshot procedure id = 70 2023-09-06T20:32:43,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=70, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1694032362036_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } in 1.3920 sec 2023-09-06T20:32:44,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=70 2023-09-06T20:32:44,156 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1694032357701 2023-09-06T20:32:44,156 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-09-06T20:32:44,156 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-09-06T20:32:44,156 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1694032357701_test-1694032296496 2023-09-06T20:32:44,156 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1694032362036_default_test-1694032296496 to hdfs://localhost:39327/backupUT/backup_1694032357701/default/test-1694032296496/ 2023-09-06T20:32:44,156 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-09-06T20:32:44,180 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:32:44,180 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37419,1694032284546_-2012493626_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:39327/backupUT/backup_1694032357701/default/test-1694032296496, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:39327/backupUT/backup_1694032357701/default/test-1694032296496/.hbase-snapshot/.tmp/snapshot_1694032362036_default_test-1694032296496 2023-09-06T20:32:44,180 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1, snapshotDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032362036_default_test-1694032296496. 2023-09-06T20:32:44,189 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032362036_default_test-1694032296496 to hdfs://localhost:39327/backupUT/backup_1694032357701/default/test-1694032296496/.hbase-snapshot/.tmp/snapshot_1694032362036_default_test-1694032296496 2023-09-06T20:32:44,608 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:44,608 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:44,609 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:44,609 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:45,581 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop-6424370499196021935.jar 2023-09-06T20:32:45,581 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:45,581 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:45,582 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:45,582 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:45,583 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:45,583 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:45,583 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:45,584 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:45,584 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:45,585 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:32:45,585 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:32:45,586 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:32:45,586 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:32:45,587 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:32:45,587 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:32:45,587 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:32:45,588 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:32:45,588 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:32:45,589 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:32:45,589 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:32:45,589 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:32:45,590 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:32:45,591 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:32:45,591 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:32:45,591 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:32:45,592 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:32:45,592 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:32:45,593 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:32:45,636 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1694032362036_default_test-1694032296496' hfile list 2023-09-06T20:32:45,638 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-09-06T20:32:45,710 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032365675/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:32:45,710 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032365675/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:32:45,765 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-09-06T20:32:45,789 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/f/399ab11276b34883846f75d55f1892df_SeqId_4_ output=hdfs://localhost:39327/backupUT/backup_1694032357701/default/test-1694032296496/archive/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/f/399ab11276b34883846f75d55f1892df_SeqId_4_ 2023-09-06T20:32:45,790 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 8.320M/sec 2023-09-06T20:32:46,746 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-09-06T20:32:46,747 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-09-06T20:32:46,754 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1694032362036_default_test-1694032296496 2023-09-06T20:32:46,754 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1694032362036_default_test-1694032296496 finished. 2023-09-06T20:32:46,754 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1014): test-1694032296496 2023-09-06T20:32:46,761 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1694032357701 2023-09-06T20:32:46,761 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-09-06T20:32:47,172 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:39327/backupUT/backup_1694032357701/.backup.manifest 2023-09-06T20:32:47,173 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(358): Backup backup_1694032357701 finished: type=FULL,tablelist=test-1694032296496,targetRootDir=hdfs://localhost:39327/backupUT,startts=1694032361849,completets=1694032366761,bytescopied=0 2023-09-06T20:32:47,173 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-09-06T20:32:47,173 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1694032362036_default_test-1694032296496 2023-09-06T20:32:47,174 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1694032362036_default_test-1694032296496" 2023-09-06T20:32:47,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1694032362036_default_test-1694032296496 2023-09-06T20:32:47,177 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1694032362036_default_test-1694032296496 for backup backup_1694032357701 succeeded. 2023-09-06T20:32:47,178 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:32:47,182 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:32:47,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:32:47,185 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:32:47,187 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:32:47,192 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(376): Backup backup_1694032357701 completed. 2023-09-06T20:32:47,193 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:32:47,204 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] backup.TestFullRestore(61): backup complete 2023-09-06T20:32:47,204 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(101): HBase table table1 does not exist. It will be created during restore process 2023-09-06T20:32:47,205 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:39327/backupUT/backup_1694032357701/.backup.manifest 2023-09-06T20:32:47,207 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032357701/.backup.manifest 2023-09-06T20:32:47,207 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:39327/backupUT/backup_1694032357701/.backup.manifest 2023-09-06T20:32:47,209 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032357701/.backup.manifest 2023-09-06T20:32:47,209 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(148): Restoring 'test-1694032296496' to 'table1' from full backup image hdfs://localhost:39327/backupUT/backup_1694032357701/default/test-1694032296496 2023-09-06T20:32:47,217 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(487): Creating target table 'table1' 2023-09-06T20:32:47,217 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:39327/backupUT/backup_1694032357701/default/test-1694032296496/archive/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454 2023-09-06T20:32:47,218 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:39327/backupUT/backup_1694032357701/default/test-1694032296496/archive/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/f in region [hdfs://localhost:39327/backupUT/backup_1694032357701/default/test-1694032296496/archive/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454] 2023-09-06T20:32:47,222 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:39327/backupUT/backup_1694032357701/default/test-1694032296496/archive/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/f/399ab11276b34883846f75d55f1892df_SeqId_4_ first=row0 last=row98 2023-09-06T20:32:47,223 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:32:47,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=table1 2023-09-06T20:32:47,226 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:32:47,226 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 72 2023-09-06T20:32:47,227 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:32:47,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-09-06T20:32:47,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-09-06T20:32:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-09-06T20:32:47,635 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => a4e6225987ac86f31e44d6ae6e723ecb, NAME => 'table1,,1694032367223.a4e6225987ac86f31e44d6ae6e723ecb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:32:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-09-06T20:32:48,042 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(888): Instantiated table1,,1694032367223.a4e6225987ac86f31e44d6ae6e723ecb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:32:48,042 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1668): Closing a4e6225987ac86f31e44d6ae6e723ecb, disabling compactions & flushes 2023-09-06T20:32:48,042 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1690): Closing region table1,,1694032367223.a4e6225987ac86f31e44d6ae6e723ecb. 2023-09-06T20:32:48,042 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1694032367223.a4e6225987ac86f31e44d6ae6e723ecb. 2023-09-06T20:32:48,042 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on table1,,1694032367223.a4e6225987ac86f31e44d6ae6e723ecb. after waiting 0 ms 2023-09-06T20:32:48,042 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region table1,,1694032367223.a4e6225987ac86f31e44d6ae6e723ecb. 2023-09-06T20:32:48,042 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1909): Closed table1,,1694032367223.a4e6225987ac86f31e44d6ae6e723ecb. 2023-09-06T20:32:48,042 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1622): Region close journal for a4e6225987ac86f31e44d6ae6e723ecb: 2023-09-06T20:32:48,043 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:32:48,044 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"table1,,1694032367223.a4e6225987ac86f31e44d6ae6e723ecb.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1694032368044"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032368044"}]},"ts":"1694032368044"} 2023-09-06T20:32:48,045 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:32:48,046 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:32:48,046 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032368046"}]},"ts":"1694032368046"} 2023-09-06T20:32:48,048 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLING in hbase:meta 2023-09-06T20:32:48,052 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=a4e6225987ac86f31e44d6ae6e723ecb, ASSIGN}] 2023-09-06T20:32:48,054 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=a4e6225987ac86f31e44d6ae6e723ecb, ASSIGN 2023-09-06T20:32:48,054 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=table1, region=a4e6225987ac86f31e44d6ae6e723ecb, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36961,1694032287907; forceNewPlan=false, retain=false 2023-09-06T20:32:48,206 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=a4e6225987ac86f31e44d6ae6e723ecb, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:48,208 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE; OpenRegionProcedure a4e6225987ac86f31e44d6ae6e723ecb, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:32:48,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-09-06T20:32:48,363 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(130): Open table1,,1694032367223.a4e6225987ac86f31e44d6ae6e723ecb. 2023-09-06T20:32:48,363 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7254): Opening region: {ENCODED => a4e6225987ac86f31e44d6ae6e723ecb, NAME => 'table1,,1694032367223.a4e6225987ac86f31e44d6ae6e723ecb.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:32:48,363 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:32:48,363 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 a4e6225987ac86f31e44d6ae6e723ecb 2023-09-06T20:32:48,363 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(888): Instantiated table1,,1694032367223.a4e6225987ac86f31e44d6ae6e723ecb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:32:48,363 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7296): checking encryption for a4e6225987ac86f31e44d6ae6e723ecb 2023-09-06T20:32:48,364 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7299): checking classloading for a4e6225987ac86f31e44d6ae6e723ecb 2023-09-06T20:32:48,365 INFO [StoreOpener-a4e6225987ac86f31e44d6ae6e723ecb-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region a4e6225987ac86f31e44d6ae6e723ecb 2023-09-06T20:32:48,366 INFO [StoreOpener-a4e6225987ac86f31e44d6ae6e723ecb-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4e6225987ac86f31e44d6ae6e723ecb columnFamilyName f 2023-09-06T20:32:48,366 DEBUG [StoreOpener-a4e6225987ac86f31e44d6ae6e723ecb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:32:48,367 INFO [StoreOpener-a4e6225987ac86f31e44d6ae6e723ecb-1 {}] regionserver.HStore(324): Store=a4e6225987ac86f31e44d6ae6e723ecb/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:32:48,367 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/table1/a4e6225987ac86f31e44d6ae6e723ecb 2023-09-06T20:32:48,368 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/table1/a4e6225987ac86f31e44d6ae6e723ecb 2023-09-06T20:32:48,370 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1076): writing seq id for a4e6225987ac86f31e44d6ae6e723ecb 2023-09-06T20:32:48,372 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/table1/a4e6225987ac86f31e44d6ae6e723ecb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:32:48,372 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): Opened a4e6225987ac86f31e44d6ae6e723ecb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10040873600, jitterRate=-0.06487077474594116}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:32:48,375 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(992): Region open journal for a4e6225987ac86f31e44d6ae6e723ecb: 2023-09-06T20:32:48,375 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2556): Post open deploy tasks for table1,,1694032367223.a4e6225987ac86f31e44d6ae6e723ecb., pid=74, masterSystemTime=1694032368359 2023-09-06T20:32:48,377 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2583): Finished post open deploy task for table1,,1694032367223.a4e6225987ac86f31e44d6ae6e723ecb. 2023-09-06T20:32:48,377 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(158): Opened table1,,1694032367223.a4e6225987ac86f31e44d6ae6e723ecb. 2023-09-06T20:32:48,377 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=a4e6225987ac86f31e44d6ae6e723ecb, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:48,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=74, resume processing ppid=73 2023-09-06T20:32:48,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=74, ppid=73, state=SUCCESS; OpenRegionProcedure a4e6225987ac86f31e44d6ae6e723ecb, server=jenkins-hbase3.apache.org,36961,1694032287907 in 171 msec 2023-09-06T20:32:48,382 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=73, resume processing ppid=72 2023-09-06T20:32:48,382 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=73, ppid=72, state=SUCCESS; TransitRegionStateProcedure table=table1, region=a4e6225987ac86f31e44d6ae6e723ecb, ASSIGN in 328 msec 2023-09-06T20:32:48,382 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:32:48,382 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032368382"}]},"ts":"1694032368382"} 2023-09-06T20:32:48,383 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLED in hbase:meta 2023-09-06T20:32:48,386 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:32:48,388 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=72, state=SUCCESS; CreateTableProcedure table=table1 in 1.1630 sec 2023-09-06T20:32:49,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-09-06T20:32:49,337 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:39327/backupUT/backup_1694032357701/default/test-1694032296496/archive/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454 from hbase tables test-1694032296496 to tables table1 2023-09-06T20:32:49,337 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1694032296496 into table1 2023-09-06T20:32:49,339 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-table1-1694032369338 from hdfs://localhost:39327/backupUT/backup_1694032357701/default/test-1694032296496/archive/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454 2023-09-06T20:32:49,345 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37cc47bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:49,346 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:32:49,347 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:43068, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:32:49,349 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-09-06T20:32:49,349 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table table1 2023-09-06T20:32:49,353 DEBUG [hconnection-0x538606c0-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:32:49,355 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45560, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:32:49,357 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for table1: 2023-09-06T20:32:49,357 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-09-06T20:32:49,357 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/partitions_5b21500c-b4ba-4eef-adc0-dd6484d92f1e 2023-09-06T20:32:49,366 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:49,367 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:49,367 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:49,368 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:50,336 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop-4095094836092570316.jar 2023-09-06T20:32:50,337 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:50,337 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:50,338 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:50,338 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:50,338 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:50,339 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:50,339 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:50,339 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:50,340 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:50,340 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:32:50,341 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:32:50,341 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:32:50,342 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:32:50,342 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:32:50,343 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:32:50,343 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:32:50,343 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:32:50,344 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:32:50,344 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:32:50,345 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:32:50,345 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:32:50,345 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:50,346 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:50,346 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:50,346 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:50,347 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:50,347 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:50,348 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:32:50,348 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: table1 2023-09-06T20:32:50,348 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:32:50,348 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-09-06T20:32:50,348 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-09-06T20:32:50,349 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:32:50,360 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-09-06T20:32:50,435 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032370402/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:32:50,435 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032370402/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:32:50,484 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:39327/backupUT/backup_1694032357701/default/test-1694032296496/archive/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/f/399ab11276b34883846f75d55f1892df_SeqId_4_ 2023-09-06T20:32:50,488 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-09-06T20:32:50,511 DEBUG [pool-299-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@740fc9f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:50,513 DEBUG [hconnection-0x668d7918-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:32:50,514 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:59608, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:32:50,515 DEBUG [pool-299-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:32:50,517 DEBUG [pool-299-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-09-06T20:32:50,517 DEBUG [pool-299-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-09-06T20:32:50,523 DEBUG [pool-299-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-table1-1694032369338/_temporary/0/_temporary/attempt_local1955928509_0009_r_000000_0/f/f5f06e839dbf4e728671626d4e20e3ac is 37, key is row10/f:q1/1694032299136/Put/seqid=0 2023-09-06T20:32:50,525 WARN [Thread-1256 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:36961] Specified favored nodes: [jenkins-hbase3.apache.org:36961] 2023-09-06T20:32:51,479 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-table1-1694032369338 2023-09-06T20:32:51,485 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@870102, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:51,487 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:32:51,488 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:59622, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:32:51,494 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-table1-1694032369338/_SUCCESS 2023-09-06T20:32:51,497 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:32:51,498 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46122, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:32:51,507 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-table1-1694032369338/f/f5f06e839dbf4e728671626d4e20e3ac first=Optional[row0] last=Optional[row98] 2023-09-06T20:32:51,508 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=table1,,1694032367223.a4e6225987ac86f31e44d6ae6e723ecb., hostname=jenkins-hbase3.apache.org,36961,1694032287907, seqNum=2 for row with hfile group [{f,hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-table1-1694032369338/f/f5f06e839dbf4e728671626d4e20e3ac}] 2023-09-06T20:32:51,512 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-table1-1694032369338/f/f5f06e839dbf4e728671626d4e20e3ac for inclusion in a4e6225987ac86f31e44d6ae6e723ecb/f 2023-09-06T20:32:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-09-06T20:32:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(625): Region bounds: first= last= 2023-09-06T20:32:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HRegion(2520): Flush status journal for a4e6225987ac86f31e44d6ae6e723ecb: 2023-09-06T20:32:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-table1-1694032369338/f/f5f06e839dbf4e728671626d4e20e3ac to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__table1__gpl5ab35r6krfqbm8ts8h9cdoivp05bnsiomj5d2hgh2ker6c77dc01fu4a93go6/f/f5f06e839dbf4e728671626d4e20e3ac 2023-09-06T20:32:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__table1__gpl5ab35r6krfqbm8ts8h9cdoivp05bnsiomj5d2hgh2ker6c77dc01fu4a93go6/f/f5f06e839dbf4e728671626d4e20e3ac as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/table1/a4e6225987ac86f31e44d6ae6e723ecb/f/68c4b72edf194b69bcc204041324fb5c_SeqId_4_ 2023-09-06T20:32:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71a4af1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:32:51,529 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46138, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:32:51,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:32:51,533 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:59638, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:32:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:32:51,547 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__table1__gpl5ab35r6krfqbm8ts8h9cdoivp05bnsiomj5d2hgh2ker6c77dc01fu4a93go6/f/f5f06e839dbf4e728671626d4e20e3ac into a4e6225987ac86f31e44d6ae6e723ecb/f as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/table1/a4e6225987ac86f31e44d6ae6e723ecb/f/68c4b72edf194b69bcc204041324fb5c_SeqId_4_ - updating store file list. 2023-09-06T20:32:51,552 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/table1/a4e6225987ac86f31e44d6ae6e723ecb/f/68c4b72edf194b69bcc204041324fb5c_SeqId_4_ into a4e6225987ac86f31e44d6ae6e723ecb/f 2023-09-06T20:32:51,552 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__table1__gpl5ab35r6krfqbm8ts8h9cdoivp05bnsiomj5d2hgh2ker6c77dc01fu4a93go6/f/f5f06e839dbf4e728671626d4e20e3ac into a4e6225987ac86f31e44d6ae6e723ecb/f (new location: hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/table1/a4e6225987ac86f31e44d6ae6e723ecb/f/68c4b72edf194b69bcc204041324fb5c_SeqId_4_) 2023-09-06T20:32:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__table1__gpl5ab35r6krfqbm8ts8h9cdoivp05bnsiomj5d2hgh2ker6c77dc01fu4a93go6/f/f5f06e839dbf4e728671626d4e20e3ac 2023-09-06T20:32:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@648b0720, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:32:51,560 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46148, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-09-06T20:32:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:32:51,565 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:59650, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-09-06T20:32:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:32:51,579 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:32:51,579 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-09-06T20:32:51,579 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-09-06T20:32:51,579 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(233): Backup: backup_1694032357701 hdfs://localhost:39327/backupUT/backup_1694032357701/default/test-1694032296496/ 2023-09-06T20:32:51,580 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-09-06T20:32:51,581 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable table1 2023-09-06T20:32:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=75, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=table1 2023-09-06T20:32:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=75 2023-09-06T20:32:51,585 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032371584"}]},"ts":"1694032371584"} 2023-09-06T20:32:51,586 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLING in hbase:meta 2023-09-06T20:32:51,587 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set table1 to state=DISABLING 2023-09-06T20:32:51,588 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=a4e6225987ac86f31e44d6ae6e723ecb, UNASSIGN}] 2023-09-06T20:32:51,589 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=a4e6225987ac86f31e44d6ae6e723ecb, UNASSIGN 2023-09-06T20:32:51,590 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=a4e6225987ac86f31e44d6ae6e723ecb, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:51,591 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE; CloseRegionProcedure a4e6225987ac86f31e44d6ae6e723ecb, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:32:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=75 2023-09-06T20:32:51,743 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(119): Close a4e6225987ac86f31e44d6ae6e723ecb 2023-09-06T20:32:51,743 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1668): Closing a4e6225987ac86f31e44d6ae6e723ecb, disabling compactions & flushes 2023-09-06T20:32:51,743 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1690): Closing region table1,,1694032367223.a4e6225987ac86f31e44d6ae6e723ecb. 2023-09-06T20:32:51,743 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1694032367223.a4e6225987ac86f31e44d6ae6e723ecb. 2023-09-06T20:32:51,743 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1778): Acquired close lock on table1,,1694032367223.a4e6225987ac86f31e44d6ae6e723ecb. after waiting 0 ms 2023-09-06T20:32:51,743 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1788): Updates disabled for region table1,,1694032367223.a4e6225987ac86f31e44d6ae6e723ecb. 2023-09-06T20:32:51,750 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/table1/a4e6225987ac86f31e44d6ae6e723ecb/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-09-06T20:32:51,750 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:32:51,750 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1909): Closed table1,,1694032367223.a4e6225987ac86f31e44d6ae6e723ecb. 2023-09-06T20:32:51,750 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1622): Region close journal for a4e6225987ac86f31e44d6ae6e723ecb: 2023-09-06T20:32:51,752 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(163): Closed a4e6225987ac86f31e44d6ae6e723ecb 2023-09-06T20:32:51,752 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=a4e6225987ac86f31e44d6ae6e723ecb, regionState=CLOSED 2023-09-06T20:32:51,755 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=77, resume processing ppid=76 2023-09-06T20:32:51,755 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=77, ppid=76, state=SUCCESS; CloseRegionProcedure a4e6225987ac86f31e44d6ae6e723ecb, server=jenkins-hbase3.apache.org,36961,1694032287907 in 162 msec 2023-09-06T20:32:51,756 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=76, resume processing ppid=75 2023-09-06T20:32:51,756 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=76, ppid=75, state=SUCCESS; TransitRegionStateProcedure table=table1, region=a4e6225987ac86f31e44d6ae6e723ecb, UNASSIGN in 167 msec 2023-09-06T20:32:51,756 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032371756"}]},"ts":"1694032371756"} 2023-09-06T20:32:51,757 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLED in hbase:meta 2023-09-06T20:32:51,759 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(305): Set table1 to state=DISABLED 2023-09-06T20:32:51,761 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=75, state=SUCCESS; DisableTableProcedure table=table1 in 179 msec 2023-09-06T20:32:51,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=75 2023-09-06T20:32:51,893 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$5(2448): Client=jenkins//172.31.12.81 delete table1 2023-09-06T20:32:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=table1 2023-09-06T20:32:51,902 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-09-06T20:32:51,903 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=78, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=table1 2023-09-06T20:32:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=78 2023-09-06T20:32:51,905 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/table1/a4e6225987ac86f31e44d6ae6e723ecb 2023-09-06T20:32:51,907 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/table1/a4e6225987ac86f31e44d6ae6e723ecb/f, FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/table1/a4e6225987ac86f31e44d6ae6e723ecb/recovered.edits] 2023-09-06T20:32:51,911 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/table1/a4e6225987ac86f31e44d6ae6e723ecb/f/68c4b72edf194b69bcc204041324fb5c_SeqId_4_ to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/archive/data/default/table1/a4e6225987ac86f31e44d6ae6e723ecb/f/68c4b72edf194b69bcc204041324fb5c_SeqId_4_ 2023-09-06T20:32:51,914 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/table1/a4e6225987ac86f31e44d6ae6e723ecb/recovered.edits/6.seqid to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/archive/data/default/table1/a4e6225987ac86f31e44d6ae6e723ecb/recovered.edits/6.seqid 2023-09-06T20:32:51,915 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/table1/a4e6225987ac86f31e44d6ae6e723ecb 2023-09-06T20:32:51,915 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(293): Archived table1 regions 2023-09-06T20:32:51,917 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=78, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=table1 2023-09-06T20:32:51,918 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of table1 from hbase:meta 2023-09-06T20:32:51,920 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(387): Removing 'table1' descriptor. 2023-09-06T20:32:51,921 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=78, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=table1 2023-09-06T20:32:51,921 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(377): Removing 'table1' from region states. 2023-09-06T20:32:51,921 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1,,1694032367223.a4e6225987ac86f31e44d6ae6e723ecb.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1694032371921"}]},"ts":"9223372036854775807"} 2023-09-06T20:32:51,923 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-06T20:32:51,923 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => a4e6225987ac86f31e44d6ae6e723ecb, NAME => 'table1,,1694032367223.a4e6225987ac86f31e44d6ae6e723ecb.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:32:51,923 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(381): Marking 'table1' as deleted. 2023-09-06T20:32:51,923 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1694032371923"}]},"ts":"9223372036854775807"} 2023-09-06T20:32:51,925 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1678): Deleted table table1 state from META 2023-09-06T20:32:51,927 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(127): Finished pid=78, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-09-06T20:32:51,928 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=78, state=SUCCESS; DeleteTableProcedure table=table1 in 33 msec 2023-09-06T20:32:52,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=78 2023-09-06T20:32:52,023 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingle Thread=595 (was 595), OpenFileDescriptor=673 (was 677), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=155 (was 169), ProcessCount=167 (was 167), AvailableMemoryMB=3177 (was 3324) 2023-09-06T20:32:52,023 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(130): Thread=595 is superior to 500 2023-09-06T20:32:52,038 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingleOverwriteCommand Thread=595, OpenFileDescriptor=673, MaxFileDescriptor=60000, SystemLoadAverage=155, ProcessCount=167, AvailableMemoryMB=3177 2023-09-06T20:32:52,038 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(130): Thread=595 is superior to 500 2023-09-06T20:32:52,038 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] backup.TestFullRestore(192): test full restore on a single table empty table: command-line 2023-09-06T20:32:52,044 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@207ed784, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:52,047 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:32:52,049 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:59654, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:32:52,051 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:32:52,052 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46160, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:32:52,067 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:32:52,072 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:32:52,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032372072 (current time:1694032372072). 2023-09-06T20:32:52,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:32:52,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:32:52,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53a72be7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:52,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:32:52,081 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:59670, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:32:52,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:32:52,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:32:52,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=79, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:32:52,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 79 2023-09-06T20:32:52,084 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:32:52,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=79 2023-09-06T20:32:52,085 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:32:52,086 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:32:52,091 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:32:52,091 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281}] 2023-09-06T20:32:52,093 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=80, ppid=79, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 2023-09-06T20:32:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=79 2023-09-06T20:32:52,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=80 2023-09-06T20:32:52,244 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:32:52,244 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HRegion(2819): Flushing cf63a85c53279bc94811b50578f31281 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-09-06T20:32:52,248 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/01fb4a2eb68c4823807811190aa11f8b is 169, key is trslm:hdfs://localhost:39327/backupUT\x00test-1694032296496/meta:log-roll-map/1694032366756/Put/seqid=0 2023-09-06T20:32:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=79 2023-09-06T20:32:52,652 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/01fb4a2eb68c4823807811190aa11f8b 2023-09-06T20:32:52,660 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/849e36507ab34f85a508bffbc6e215d3 is 452, key is session:backup_1694032357701/session:context/1694032367186/Put/seqid=0 2023-09-06T20:32:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=79 2023-09-06T20:32:53,065 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/849e36507ab34f85a508bffbc6e215d3 2023-09-06T20:32:53,071 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/01fb4a2eb68c4823807811190aa11f8b as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/01fb4a2eb68c4823807811190aa11f8b 2023-09-06T20:32:53,076 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/01fb4a2eb68c4823807811190aa11f8b, entries=4, sequenceid=61, filesize=5.5 K 2023-09-06T20:32:53,077 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/849e36507ab34f85a508bffbc6e215d3 as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/849e36507ab34f85a508bffbc6e215d3 2023-09-06T20:32:53,082 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/849e36507ab34f85a508bffbc6e215d3, entries=2, sequenceid=61, filesize=5.4 K 2023-09-06T20:32:53,083 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for cf63a85c53279bc94811b50578f31281 in 839ms, sequenceid=61, compaction requested=true 2023-09-06T20:32:53,083 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HRegion(2520): Flush status journal for cf63a85c53279bc94811b50578f31281: 2023-09-06T20:32:53,083 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. for snapshot_backup_system completed. 2023-09-06T20:32:53,083 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:32:53,083 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:32:53,083 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/0379b1e1f59e40d2ac2991b2ae6b86ce, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/dd7edbeade334fa7895ff079e0fc3472, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/78eec26cfa0242e48353b19a4a00b7d5, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/faffbcb72f8649bcad705936a3105fd4, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/01fb4a2eb68c4823807811190aa11f8b] hfiles 2023-09-06T20:32:53,083 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/0379b1e1f59e40d2ac2991b2ae6b86ce for snapshot=snapshot_backup_system 2023-09-06T20:32:53,083 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/dd7edbeade334fa7895ff079e0fc3472 for snapshot=snapshot_backup_system 2023-09-06T20:32:53,083 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/78eec26cfa0242e48353b19a4a00b7d5 for snapshot=snapshot_backup_system 2023-09-06T20:32:53,083 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/faffbcb72f8649bcad705936a3105fd4 for snapshot=snapshot_backup_system 2023-09-06T20:32:53,083 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/01fb4a2eb68c4823807811190aa11f8b for snapshot=snapshot_backup_system 2023-09-06T20:32:53,083 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/4ced064cdbd34bd6b594c76f049fa4ee, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/baca9a30045a4dabad9b3b78a8fd6c86, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/bcfde09dc46641bebf14d756f321d0af, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/2dc68dbd9f3749a1a2d9d7333fa4771d, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/6da826e7b47e41daa41e268fb4e951b2, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/849e36507ab34f85a508bffbc6e215d3] hfiles 2023-09-06T20:32:53,083 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (1/6): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/4ced064cdbd34bd6b594c76f049fa4ee for snapshot=snapshot_backup_system 2023-09-06T20:32:53,083 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (2/6): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/baca9a30045a4dabad9b3b78a8fd6c86 for snapshot=snapshot_backup_system 2023-09-06T20:32:53,083 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (3/6): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/bcfde09dc46641bebf14d756f321d0af for snapshot=snapshot_backup_system 2023-09-06T20:32:53,083 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (4/6): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/2dc68dbd9f3749a1a2d9d7333fa4771d for snapshot=snapshot_backup_system 2023-09-06T20:32:53,083 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (5/6): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/6da826e7b47e41daa41e268fb4e951b2 for snapshot=snapshot_backup_system 2023-09-06T20:32:53,083 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (6/6): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/849e36507ab34f85a508bffbc6e215d3 for snapshot=snapshot_backup_system 2023-09-06T20:32:53,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=79 2023-09-06T20:32:53,489 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:32:53,489 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=80 2023-09-06T20:32:53,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=80 2023-09-06T20:32:53,489 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region cf63a85c53279bc94811b50578f31281 2023-09-06T20:32:53,491 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=80, ppid=79, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 2023-09-06T20:32:53,493 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=80, resume processing ppid=79 2023-09-06T20:32:53,493 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:32:53,493 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=80, ppid=79, state=SUCCESS; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 in 1.4000 sec 2023-09-06T20:32:53,493 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:32:53,494 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:32:53,494 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:32:53,495 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:32:53,905 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:32:53,908 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:32:53,917 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:32:53,917 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:32:53,919 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:32:53,919 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 79 2023-09-06T20:32:53,921 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=79, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8370 sec 2023-09-06T20:32:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=79 2023-09-06T20:32:54,190 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(115): Backup backup_1694032372044 started at 1694032374190. 2023-09-06T20:32:54,192 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(119): Backup session backup_1694032372044 has been started. 2023-09-06T20:32:54,197 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-09-06T20:32:54,197 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-09-06T20:32:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-09-06T20:32:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-09-06T20:32:54,198 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-09-06T20:32:54,198 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-09-06T20:32:54,199 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-09-06T20:32:54,199 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-09-06T20:32:54,199 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:32:54,206 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:54,206 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:32:54,206 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:32:54,207 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-09-06T20:32:54,207 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-09-06T20:32:54,207 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-09-06T20:32:54,207 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:32:54,207 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-09-06T20:32:54,208 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-09-06T20:32:54,208 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-09-06T20:32:54,208 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-09-06T20:32:54,208 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-09-06T20:32:54,208 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-09-06T20:32:54,209 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-09-06T20:32:54,209 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-09-06T20:32:54,209 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,36961,1694032287907' joining acquired barrier for procedure (rolllog) in zk 2023-09-06T20:32:54,211 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-09-06T20:32:54,211 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:54,211 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:32:54,211 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:32:54,211 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-09-06T20:32:54,211 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:32:54,212 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:32:54,212 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:32:54,212 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:54,212 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:32:54,213 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,36961,1694032287907' joining acquired barrier for procedure 'rolllog' on coordinator 2023-09-06T20:32:54,213 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@4de63b3c[Count = 0] remaining members to acquire global barrier 2023-09-06T20:32:54,213 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-09-06T20:32:54,213 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-09-06T20:32:54,214 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-09-06T20:32:54,214 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-09-06T20:32:54,214 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-09-06T20:32:54,214 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-09-06T20:32:54,214 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-09-06T20:32:54,214 DEBUG [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:54,214 INFO [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1694032361884 highest: 1694032361884 on jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:54,214 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta:.meta(num 1694032361867) roll requested 2023-09-06T20:32:54,226 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032374214.meta, exclude list is [], retry=0 2023-09-06T20:32:54,231 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK] 2023-09-06T20:32:54,233 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032361867.meta with entries=12, filesize=2.73 KB; new WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032374214.meta 2023-09-06T20:32:54,233 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK]] 2023-09-06T20:32:54,233 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032361867.meta is not closed yet, will try archiving it next time 2023-09-06T20:32:54,233 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36961%2C1694032287907:(num 1694032361884) roll requested 2023-09-06T20:32:54,245 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032374233, exclude list is [], retry=0 2023-09-06T20:32:54,248 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK] 2023-09-06T20:32:54,249 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032361884 with entries=13, filesize=3.09 KB; new WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032374233 2023-09-06T20:32:54,249 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK]] 2023-09-06T20:32:54,250 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032361884 is not closed yet, will try archiving it next time 2023-09-06T20:32:54,250 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032352318 to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/oldWALs/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032352318 2023-09-06T20:32:54,254 DEBUG [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-09-06T20:32:54,254 INFO [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1694032374233 on jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:54,263 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-09-06T20:32:54,263 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,36961,1694032287907' in zk 2023-09-06T20:32:54,265 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-09-06T20:32:54,265 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:32:54,265 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-09-06T20:32:54,265 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:54,266 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:32:54,266 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:32:54,267 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:32:54,267 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:32:54,267 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:32:54,267 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:54,268 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:32:54,268 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:32:54,268 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:54,269 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,36961,1694032287907': 2023-09-06T20:32:54,269 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,36961,1694032287907' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-09-06T20:32:54,269 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-09-06T20:32:54,269 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-09-06T20:32:54,269 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-09-06T20:32:54,269 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-09-06T20:32:54,269 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-09-06T20:32:54,271 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-09-06T20:32:54,271 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-09-06T20:32:54,271 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:32:54,272 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:32:54,272 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:32:54,272 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:32:54,272 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:32:54,272 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:32:54,272 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:32:54,272 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:32:54,272 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:32:54,273 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:32:54,273 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:54,273 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:32:54,274 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:32:54,274 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:54,277 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:32:54,277 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:32:54,277 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:32:54,277 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-09-06T20:32:54,277 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-09-06T20:32:54,278 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:32:54,278 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:32:54,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-09-06T20:32:54,382 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1694032374381_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } 2023-09-06T20:32:54,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032374382 (current time:1694032374382). 2023-09-06T20:32:54,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:32:54,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1694032374381_default_test-1694032296496 VERSION not specified, setting to 2 2023-09-06T20:32:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d297e25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:54,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:32:54,392 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:59674, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:32:54,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:32:54,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:32:54,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=81, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1694032374381_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } 2023-09-06T20:32:54,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1694032374381_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 }, snapshot procedure id = 81 2023-09-06T20:32:54,395 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1694032374381_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:32:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=81 2023-09-06T20:32:54,396 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1694032374381_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:32:54,398 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1694032374381_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:32:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=81 2023-09-06T20:32:54,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=81 2023-09-06T20:32:54,803 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1694032374381_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:32:54,804 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE; SnapshotRegionProcedure f63c4b76a967620ba02f0f167d7de454}] 2023-09-06T20:32:54,805 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=82, ppid=81, state=RUNNABLE; SnapshotRegionProcedure f63c4b76a967620ba02f0f167d7de454 2023-09-06T20:32:54,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=82 2023-09-06T20:32:54,957 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454. 2023-09-06T20:32:54,957 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] regionserver.HRegion(2520): Flush status journal for f63c4b76a967620ba02f0f167d7de454: 2023-09-06T20:32:54,957 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454. for snapshot_1694032374381_default_test-1694032296496 completed. 2023-09-06T20:32:54,957 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] snapshot.SnapshotManifest(240): Storing 'test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454.' region-info for snapshot=snapshot_1694032374381_default_test-1694032296496 2023-09-06T20:32:54,957 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:32:54,957 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/f/399ab11276b34883846f75d55f1892df_SeqId_4_] hfiles 2023-09-06T20:32:54,957 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/f/399ab11276b34883846f75d55f1892df_SeqId_4_ for snapshot=snapshot_1694032374381_default_test-1694032296496 2023-09-06T20:32:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=81 2023-09-06T20:32:55,363 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454. 2023-09-06T20:32:55,363 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=82 2023-09-06T20:32:55,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=82 2023-09-06T20:32:55,364 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1694032374381_default_test-1694032296496 on region f63c4b76a967620ba02f0f167d7de454 2023-09-06T20:32:55,365 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=82, ppid=81, state=RUNNABLE; SnapshotRegionProcedure f63c4b76a967620ba02f0f167d7de454 2023-09-06T20:32:55,367 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=82, resume processing ppid=81 2023-09-06T20:32:55,367 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=82, ppid=81, state=SUCCESS; SnapshotRegionProcedure f63c4b76a967620ba02f0f167d7de454 in 563 msec 2023-09-06T20:32:55,367 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1694032374381_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:32:55,368 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1694032374381_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:32:55,368 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1694032374381_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:32:55,368 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1694032374381_default_test-1694032296496 2023-09-06T20:32:55,369 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032374381_default_test-1694032296496 2023-09-06T20:32:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=81 2023-09-06T20:32:55,778 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1694032374381_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:32:55,784 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1694032374381_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:32:55,784 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032374381_default_test-1694032296496 to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032374381_default_test-1694032296496 2023-09-06T20:32:55,785 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1694032374381_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:32:55,785 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1694032374381_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 }, snapshot procedure id = 81 2023-09-06T20:32:55,786 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=81, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1694032374381_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } in 1.3920 sec 2023-09-06T20:32:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=81 2023-09-06T20:32:56,502 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1694032372044 2023-09-06T20:32:56,502 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-09-06T20:32:56,502 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-09-06T20:32:56,502 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1694032372044_test-1694032296496 2023-09-06T20:32:56,502 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1694032374381_default_test-1694032296496 to hdfs://localhost:39327/backupUT/backup_1694032372044/default/test-1694032296496/ 2023-09-06T20:32:56,502 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-09-06T20:32:56,525 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:32:56,525 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37419,1694032284546_789536373_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:39327/backupUT/backup_1694032372044/default/test-1694032296496, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:39327/backupUT/backup_1694032372044/default/test-1694032296496/.hbase-snapshot/.tmp/snapshot_1694032374381_default_test-1694032296496 2023-09-06T20:32:56,525 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1, snapshotDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032374381_default_test-1694032296496. 2023-09-06T20:32:56,533 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032374381_default_test-1694032296496 to hdfs://localhost:39327/backupUT/backup_1694032372044/default/test-1694032296496/.hbase-snapshot/.tmp/snapshot_1694032374381_default_test-1694032296496 2023-09-06T20:32:56,953 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:56,954 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:56,954 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:56,954 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:57,921 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop-491256472245356745.jar 2023-09-06T20:32:57,921 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:57,922 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:57,922 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:57,923 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:57,923 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:57,923 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:57,923 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:57,924 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:57,924 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:32:57,925 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:32:57,925 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:32:57,926 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:32:57,926 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:32:57,927 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:32:57,927 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:32:57,927 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:32:57,928 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:32:57,928 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:32:57,928 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:32:57,929 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:32:57,929 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:32:57,930 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:32:57,930 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:32:57,931 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:32:57,931 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:32:57,931 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:32:57,932 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:32:57,932 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:32:57,996 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1694032374381_default_test-1694032296496' hfile list 2023-09-06T20:32:57,998 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-09-06T20:32:58,065 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032378033/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:32:58,066 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032378033/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:32:58,124 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-09-06T20:32:58,149 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/f/399ab11276b34883846f75d55f1892df_SeqId_4_ output=hdfs://localhost:39327/backupUT/backup_1694032372044/default/test-1694032296496/archive/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/f/399ab11276b34883846f75d55f1892df_SeqId_4_ 2023-09-06T20:32:58,149 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 8.320M/sec 2023-09-06T20:32:59,103 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-09-06T20:32:59,104 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-09-06T20:32:59,110 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1694032374381_default_test-1694032296496 2023-09-06T20:32:59,110 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1694032374381_default_test-1694032296496 finished. 2023-09-06T20:32:59,110 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1014): test-1694032296496 2023-09-06T20:32:59,117 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1694032372044 2023-09-06T20:32:59,117 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-09-06T20:32:59,522 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:39327/backupUT/backup_1694032372044/.backup.manifest 2023-09-06T20:32:59,522 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(358): Backup backup_1694032372044 finished: type=FULL,tablelist=test-1694032296496,targetRootDir=hdfs://localhost:39327/backupUT,startts=1694032374190,completets=1694032379116,bytescopied=0 2023-09-06T20:32:59,522 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-09-06T20:32:59,522 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1694032374381_default_test-1694032296496 2023-09-06T20:32:59,523 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1694032374381_default_test-1694032296496" 2023-09-06T20:32:59,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1694032374381_default_test-1694032296496 2023-09-06T20:32:59,527 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1694032374381_default_test-1694032296496 for backup backup_1694032372044 succeeded. 2023-09-06T20:32:59,527 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:32:59,533 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:32:59,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:32:59,535 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:32:59,537 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:32:59,543 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(376): Backup backup_1694032372044 completed. 2023-09-06T20:32:59,543 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:32:59,560 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] backup.TestFullRestore(196): backup complete 2023-09-06T20:32:59,560 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] backup.RestoreDriver(101): Found -overwrite option in restore command, will overwrite to existing table if any in the restore target 2023-09-06T20:32:59,569 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d29a158, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:32:59,571 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:32:59,575 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:59686, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:32:59,578 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:39327/backupUT/backup_1694032372044/.backup.manifest 2023-09-06T20:32:59,580 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032372044/.backup.manifest 2023-09-06T20:32:59,581 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:39327/backupUT/backup_1694032372044/.backup.manifest 2023-09-06T20:32:59,583 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032372044/.backup.manifest 2023-09-06T20:32:59,583 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(148): Restoring 'test-1694032296496' to 'test-1694032296496' from full backup image hdfs://localhost:39327/backupUT/backup_1694032372044/default/test-1694032296496 2023-09-06T20:32:59,590 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(476): Truncating exising target table 'test-1694032296496', preserving region splits 2023-09-06T20:32:59,590 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:32:59,592 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46172, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:32:59,593 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable test-1694032296496 2023-09-06T20:32:59,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=83, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=test-1694032296496 2023-09-06T20:32:59,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=83 2023-09-06T20:32:59,596 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1694032296496","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032379596"}]},"ts":"1694032379596"} 2023-09-06T20:32:59,597 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1694032296496, state=DISABLING in hbase:meta 2023-09-06T20:32:59,599 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set test-1694032296496 to state=DISABLING 2023-09-06T20:32:59,599 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1694032296496, region=f63c4b76a967620ba02f0f167d7de454, UNASSIGN}] 2023-09-06T20:32:59,600 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1694032296496, region=f63c4b76a967620ba02f0f167d7de454, UNASSIGN 2023-09-06T20:32:59,601 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=84 updating hbase:meta row=f63c4b76a967620ba02f0f167d7de454, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:32:59,602 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE; CloseRegionProcedure f63c4b76a967620ba02f0f167d7de454, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:32:59,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=83 2023-09-06T20:32:59,754 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(119): Close f63c4b76a967620ba02f0f167d7de454 2023-09-06T20:32:59,754 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1668): Closing f63c4b76a967620ba02f0f167d7de454, disabling compactions & flushes 2023-09-06T20:32:59,754 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1690): Closing region test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454. 2023-09-06T20:32:59,754 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454. 2023-09-06T20:32:59,754 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1778): Acquired close lock on test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454. after waiting 0 ms 2023-09-06T20:32:59,754 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1788): Updates disabled for region test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454. 2023-09-06T20:32:59,761 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/recovered.edits/10.seqid, newMaxSeqId=10, maxSeqId=1 2023-09-06T20:32:59,761 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:32:59,762 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1909): Closed test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454. 2023-09-06T20:32:59,762 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1622): Region close journal for f63c4b76a967620ba02f0f167d7de454: 2023-09-06T20:32:59,764 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(163): Closed f63c4b76a967620ba02f0f167d7de454 2023-09-06T20:32:59,764 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=84 updating hbase:meta row=f63c4b76a967620ba02f0f167d7de454, regionState=CLOSED 2023-09-06T20:32:59,767 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=85, resume processing ppid=84 2023-09-06T20:32:59,767 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=85, ppid=84, state=SUCCESS; CloseRegionProcedure f63c4b76a967620ba02f0f167d7de454, server=jenkins-hbase3.apache.org,36961,1694032287907 in 163 msec 2023-09-06T20:32:59,768 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=84, resume processing ppid=83 2023-09-06T20:32:59,768 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=84, ppid=83, state=SUCCESS; TransitRegionStateProcedure table=test-1694032296496, region=f63c4b76a967620ba02f0f167d7de454, UNASSIGN in 168 msec 2023-09-06T20:32:59,768 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1694032296496","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032379768"}]},"ts":"1694032379768"} 2023-09-06T20:32:59,769 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1694032296496, state=DISABLED in hbase:meta 2023-09-06T20:32:59,772 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(305): Set test-1694032296496 to state=DISABLED 2023-09-06T20:32:59,773 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=83, state=SUCCESS; DisableTableProcedure table=test-1694032296496 in 180 msec 2023-09-06T20:32:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=83 2023-09-06T20:32:59,900 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$6(2480): Client=jenkins//172.31.12.81 truncate test-1694032296496 2023-09-06T20:32:59,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=86, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=test-1694032296496 preserveSplits=true) 2023-09-06T20:32:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=86 2023-09-06T20:32:59,903 DEBUG [PEWorker-5 {}] procedure.TruncateTableProcedure(87): waiting for 'test-1694032296496' regions in transition 2023-09-06T20:32:59,909 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454 2023-09-06T20:32:59,910 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/f, FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/recovered.edits] 2023-09-06T20:32:59,914 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/f/399ab11276b34883846f75d55f1892df_SeqId_4_ to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/archive/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/f/399ab11276b34883846f75d55f1892df_SeqId_4_ 2023-09-06T20:32:59,917 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/recovered.edits/10.seqid to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/archive/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/recovered.edits/10.seqid 2023-09-06T20:32:59,917 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454 2023-09-06T20:32:59,917 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(293): Archived test-1694032296496 regions 2023-09-06T20:32:59,922 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of test-1694032296496 from hbase:meta 2023-09-06T20:32:59,923 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(387): Removing 'test-1694032296496' descriptor. 2023-09-06T20:32:59,924 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(377): Removing 'test-1694032296496' from region states. 2023-09-06T20:32:59,924 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1694032379924"}]},"ts":"9223372036854775807"} 2023-09-06T20:32:59,925 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-06T20:32:59,925 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => f63c4b76a967620ba02f0f167d7de454, NAME => 'test-1694032296496,,1694032335228.f63c4b76a967620ba02f0f167d7de454.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:32:59,925 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(381): Marking 'test-1694032296496' as deleted. 2023-09-06T20:32:59,925 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"test-1694032296496","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1694032379925"}]},"ts":"9223372036854775807"} 2023-09-06T20:32:59,927 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1678): Deleted table test-1694032296496 state from META 2023-09-06T20:33:00,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=86 2023-09-06T20:33:00,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=86 2023-09-06T20:33:00,336 INFO [RegionOpenAndInit-test-1694032296496-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 1f072a702e0732d4b296550713fb6262, NAME => 'test-1694032296496,,1694032379919.1f072a702e0732d4b296550713fb6262.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1694032296496', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:33:00,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=86 2023-09-06T20:33:00,742 DEBUG [RegionOpenAndInit-test-1694032296496-pool-0 {}] regionserver.HRegion(888): Instantiated test-1694032296496,,1694032379919.1f072a702e0732d4b296550713fb6262.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:33:00,742 DEBUG [RegionOpenAndInit-test-1694032296496-pool-0 {}] regionserver.HRegion(1668): Closing 1f072a702e0732d4b296550713fb6262, disabling compactions & flushes 2023-09-06T20:33:00,742 INFO [RegionOpenAndInit-test-1694032296496-pool-0 {}] regionserver.HRegion(1690): Closing region test-1694032296496,,1694032379919.1f072a702e0732d4b296550713fb6262. 2023-09-06T20:33:00,742 DEBUG [RegionOpenAndInit-test-1694032296496-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1694032296496,,1694032379919.1f072a702e0732d4b296550713fb6262. 2023-09-06T20:33:00,742 DEBUG [RegionOpenAndInit-test-1694032296496-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1694032296496,,1694032379919.1f072a702e0732d4b296550713fb6262. after waiting 0 ms 2023-09-06T20:33:00,742 DEBUG [RegionOpenAndInit-test-1694032296496-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1694032296496,,1694032379919.1f072a702e0732d4b296550713fb6262. 2023-09-06T20:33:00,742 INFO [RegionOpenAndInit-test-1694032296496-pool-0 {}] regionserver.HRegion(1909): Closed test-1694032296496,,1694032379919.1f072a702e0732d4b296550713fb6262. 2023-09-06T20:33:00,742 DEBUG [RegionOpenAndInit-test-1694032296496-pool-0 {}] regionserver.HRegion(1622): Region close journal for 1f072a702e0732d4b296550713fb6262: 2023-09-06T20:33:00,743 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1694032296496,,1694032379919.1f072a702e0732d4b296550713fb6262.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1694032380743"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032380743"}]},"ts":"1694032380743"} 2023-09-06T20:33:00,745 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:33:00,745 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1694032296496","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032380745"}]},"ts":"1694032380745"} 2023-09-06T20:33:00,746 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1694032296496, state=ENABLING in hbase:meta 2023-09-06T20:33:00,751 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1694032296496, region=1f072a702e0732d4b296550713fb6262, ASSIGN}] 2023-09-06T20:33:00,752 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=87, ppid=86, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1694032296496, region=1f072a702e0732d4b296550713fb6262, ASSIGN 2023-09-06T20:33:00,753 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=87, ppid=86, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1694032296496, region=1f072a702e0732d4b296550713fb6262, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36961,1694032287907; forceNewPlan=false, retain=false 2023-09-06T20:33:00,905 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=87 updating hbase:meta row=1f072a702e0732d4b296550713fb6262, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:00,906 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE; OpenRegionProcedure 1f072a702e0732d4b296550713fb6262, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:33:01,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=86 2023-09-06T20:33:01,061 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(130): Open test-1694032296496,,1694032379919.1f072a702e0732d4b296550713fb6262. 2023-09-06T20:33:01,061 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7254): Opening region: {ENCODED => 1f072a702e0732d4b296550713fb6262, NAME => 'test-1694032296496,,1694032379919.1f072a702e0732d4b296550713fb6262.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:33:01,062 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:33:01,062 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1694032296496 1f072a702e0732d4b296550713fb6262 2023-09-06T20:33:01,062 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(888): Instantiated test-1694032296496,,1694032379919.1f072a702e0732d4b296550713fb6262.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:33:01,062 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7296): checking encryption for 1f072a702e0732d4b296550713fb6262 2023-09-06T20:33:01,062 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7299): checking classloading for 1f072a702e0732d4b296550713fb6262 2023-09-06T20:33:01,063 INFO [StoreOpener-1f072a702e0732d4b296550713fb6262-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 1f072a702e0732d4b296550713fb6262 2023-09-06T20:33:01,065 INFO [StoreOpener-1f072a702e0732d4b296550713fb6262-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f072a702e0732d4b296550713fb6262 columnFamilyName f 2023-09-06T20:33:01,065 DEBUG [StoreOpener-1f072a702e0732d4b296550713fb6262-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:33:01,065 INFO [StoreOpener-1f072a702e0732d4b296550713fb6262-1 {}] regionserver.HStore(324): Store=1f072a702e0732d4b296550713fb6262/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:33:01,066 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/1f072a702e0732d4b296550713fb6262 2023-09-06T20:33:01,066 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/1f072a702e0732d4b296550713fb6262 2023-09-06T20:33:01,068 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1076): writing seq id for 1f072a702e0732d4b296550713fb6262 2023-09-06T20:33:01,070 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/1f072a702e0732d4b296550713fb6262/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:33:01,070 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1093): Opened 1f072a702e0732d4b296550713fb6262; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10399541920, jitterRate=-0.03146718442440033}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:33:01,071 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(992): Region open journal for 1f072a702e0732d4b296550713fb6262: 2023-09-06T20:33:01,072 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1694032296496,,1694032379919.1f072a702e0732d4b296550713fb6262., pid=88, masterSystemTime=1694032381058 2023-09-06T20:33:01,073 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1694032296496,,1694032379919.1f072a702e0732d4b296550713fb6262. 2023-09-06T20:33:01,073 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(158): Opened test-1694032296496,,1694032379919.1f072a702e0732d4b296550713fb6262. 2023-09-06T20:33:01,074 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=87 updating hbase:meta row=1f072a702e0732d4b296550713fb6262, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:01,077 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=88, resume processing ppid=87 2023-09-06T20:33:01,077 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=88, ppid=87, state=SUCCESS; OpenRegionProcedure 1f072a702e0732d4b296550713fb6262, server=jenkins-hbase3.apache.org,36961,1694032287907 in 169 msec 2023-09-06T20:33:01,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=87, resume processing ppid=86 2023-09-06T20:33:01,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=87, ppid=86, state=SUCCESS; TransitRegionStateProcedure table=test-1694032296496, region=1f072a702e0732d4b296550713fb6262, ASSIGN in 326 msec 2023-09-06T20:33:01,078 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1694032296496","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032381078"}]},"ts":"1694032381078"} 2023-09-06T20:33:01,080 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1694032296496, state=ENABLED in hbase:meta 2023-09-06T20:33:01,081 DEBUG [PEWorker-1 {}] procedure.TruncateTableProcedure(145): truncate 'test-1694032296496' completed 2023-09-06T20:33:01,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=86, state=SUCCESS; TruncateTableProcedure (table=test-1694032296496 preserveSplits=true) in 1.1810 sec 2023-09-06T20:33:02,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=86 2023-09-06T20:33:02,014 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:39327/backupUT/backup_1694032372044/default/test-1694032296496/archive/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454 from hbase tables test-1694032296496 to tables test-1694032296496 2023-09-06T20:33:02,014 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1694032296496 into test-1694032296496 2023-09-06T20:33:02,016 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-test-1694032296496-1694032382014 from hdfs://localhost:39327/backupUT/backup_1694032372044/default/test-1694032296496/archive/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454 2023-09-06T20:33:02,022 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7500fdff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:02,023 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:33:02,024 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:53532, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:33:02,025 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-09-06T20:33:02,025 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1694032296496 2023-09-06T20:33:02,028 DEBUG [hconnection-0x7baac279-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:02,029 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:38790, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:33:02,030 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1694032296496: 2023-09-06T20:33:02,031 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-09-06T20:33:02,031 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/partitions_79bb619b-81a1-498e-95da-57d7f9d57250 2023-09-06T20:33:02,437 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:02,438 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:02,438 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:02,438 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:03,405 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop-1561386271617560416.jar 2023-09-06T20:33:03,405 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:03,406 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:03,406 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:03,407 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:03,407 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:03,407 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:03,408 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:03,408 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:03,408 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:03,409 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:33:03,410 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:33:03,410 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:33:03,410 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:33:03,411 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:33:03,411 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:33:03,412 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:33:03,412 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:33:03,412 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:33:03,413 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:33:03,413 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:33:03,414 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:33:03,414 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:03,415 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:03,415 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:03,415 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:03,415 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:03,416 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:03,416 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:33:03,417 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: test-1694032296496 2023-09-06T20:33:03,417 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:33:03,417 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-09-06T20:33:03,417 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-09-06T20:33:03,418 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:33:03,428 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-09-06T20:33:03,506 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032383471/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:33:03,506 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032383471/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:33:03,554 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:39327/backupUT/backup_1694032372044/default/test-1694032296496/archive/data/default/test-1694032296496/f63c4b76a967620ba02f0f167d7de454/f/399ab11276b34883846f75d55f1892df_SeqId_4_ 2023-09-06T20:33:03,558 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-09-06T20:33:03,579 DEBUG [pool-331-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34d25d7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:03,581 DEBUG [hconnection-0x5d59a3cf-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:03,583 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:38792, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:33:03,588 DEBUG [pool-331-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:33:03,588 DEBUG [pool-331-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-09-06T20:33:03,588 DEBUG [pool-331-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-09-06T20:33:03,594 DEBUG [pool-331-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-test-1694032296496-1694032382014/_temporary/0/_temporary/attempt_local1118619028_0011_r_000000_0/f/233d07eabe5741c78f3f1e8a17df56fb is 37, key is row10/f:q1/1694032299136/Put/seqid=0 2023-09-06T20:33:03,596 WARN [Thread-1396 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:36961] Specified favored nodes: [jenkins-hbase3.apache.org:36961] 2023-09-06T20:33:04,550 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-test-1694032296496-1694032382014 2023-09-06T20:33:04,556 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4aa91242, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:04,557 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:04,558 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:38802, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:33:04,564 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-test-1694032296496-1694032382014/_SUCCESS 2023-09-06T20:33:04,567 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:33:04,569 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:53544, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:33:04,579 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-test-1694032296496-1694032382014/f/233d07eabe5741c78f3f1e8a17df56fb first=Optional[row0] last=Optional[row98] 2023-09-06T20:33:04,580 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=test-1694032296496,,1694032379919.1f072a702e0732d4b296550713fb6262., hostname=jenkins-hbase3.apache.org,36961,1694032287907, seqNum=2 for row with hfile group [{f,hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-test-1694032296496-1694032382014/f/233d07eabe5741c78f3f1e8a17df56fb}] 2023-09-06T20:33:04,588 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-test-1694032296496-1694032382014/f/233d07eabe5741c78f3f1e8a17df56fb for inclusion in 1f072a702e0732d4b296550713fb6262/f 2023-09-06T20:33:04,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-09-06T20:33:04,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(625): Region bounds: first= last= 2023-09-06T20:33:04,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HRegion(2520): Flush status journal for 1f072a702e0732d4b296550713fb6262: 2023-09-06T20:33:04,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-test-1694032296496-1694032382014/f/233d07eabe5741c78f3f1e8a17df56fb to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__test-1694032296496__8s37con60nsckt52koq8k1p3l12iuvvnpo062olg5od59bkk927u4nroj57oo8u0/f/233d07eabe5741c78f3f1e8a17df56fb 2023-09-06T20:33:04,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__test-1694032296496__8s37con60nsckt52koq8k1p3l12iuvvnpo062olg5od59bkk927u4nroj57oo8u0/f/233d07eabe5741c78f3f1e8a17df56fb as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/1f072a702e0732d4b296550713fb6262/f/8c33b1e0971346619bc1707bbedeefce_SeqId_4_ 2023-09-06T20:33:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af280ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:04,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:33:04,609 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:53550, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:33:04,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:04,614 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:38814, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:33:04,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] impl.BackupSystemTable(408): write bulk load descriptor to backup test-1694032296496 with 1 entries 2023-09-06T20:33:04,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] impl.BackupSystemTable(1582): writing raw bulk path hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/1f072a702e0732d4b296550713fb6262/f/8c33b1e0971346619bc1707bbedeefce_SeqId_4_ for test-1694032296496 1f072a702e0732d4b296550713fb6262 2023-09-06T20:33:04,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] impl.BackupSystemTable(415): written 1 rows for bulk load of test-1694032296496 2023-09-06T20:33:04,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:33:04,630 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__test-1694032296496__8s37con60nsckt52koq8k1p3l12iuvvnpo062olg5od59bkk927u4nroj57oo8u0/f/233d07eabe5741c78f3f1e8a17df56fb into 1f072a702e0732d4b296550713fb6262/f as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/1f072a702e0732d4b296550713fb6262/f/8c33b1e0971346619bc1707bbedeefce_SeqId_4_ - updating store file list. 2023-09-06T20:33:04,635 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/1f072a702e0732d4b296550713fb6262/f/8c33b1e0971346619bc1707bbedeefce_SeqId_4_ into 1f072a702e0732d4b296550713fb6262/f 2023-09-06T20:33:04,635 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__test-1694032296496__8s37con60nsckt52koq8k1p3l12iuvvnpo062olg5od59bkk927u4nroj57oo8u0/f/233d07eabe5741c78f3f1e8a17df56fb into 1f072a702e0732d4b296550713fb6262/f (new location: hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/1f072a702e0732d4b296550713fb6262/f/8c33b1e0971346619bc1707bbedeefce_SeqId_4_) 2023-09-06T20:33:04,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__test-1694032296496__8s37con60nsckt52koq8k1p3l12iuvvnpo062olg5od59bkk927u4nroj57oo8u0/f/233d07eabe5741c78f3f1e8a17df56fb 2023-09-06T20:33:04,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@155faac1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:04,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:33:04,645 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:53556, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-09-06T20:33:04,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:04,650 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:38820, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-09-06T20:33:04,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] impl.BackupSystemTable(388): write bulk load descriptor to backup test-1694032296496 with 1 entries 2023-09-06T20:33:04,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] impl.BackupSystemTable(1503): writing done bulk path hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/1f072a702e0732d4b296550713fb6262/f/8c33b1e0971346619bc1707bbedeefce_SeqId_4_ for test-1694032296496 1f072a702e0732d4b296550713fb6262 2023-09-06T20:33:04,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] impl.BackupSystemTable(394): written 1 rows for bulk load of test-1694032296496 2023-09-06T20:33:04,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:33:04,671 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:33:04,671 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-09-06T20:33:04,671 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-09-06T20:33:04,671 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(233): Backup: backup_1694032372044 hdfs://localhost:39327/backupUT/backup_1694032372044/default/test-1694032296496/ 2023-09-06T20:33:04,671 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-09-06T20:33:04,671 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:33:04,691 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingleOverwriteCommand Thread=604 (was 595) - Thread LEAK? -, OpenFileDescriptor=694 (was 673) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=151 (was 155), ProcessCount=167 (was 167), AvailableMemoryMB=3201 (was 3177) - AvailableMemoryMB LEAK? - 2023-09-06T20:33:04,692 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(130): Thread=604 is superior to 500 2023-09-06T20:33:04,707 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultipleDNECommand Thread=604, OpenFileDescriptor=694, MaxFileDescriptor=60000, SystemLoadAverage=151, ProcessCount=167, AvailableMemoryMB=3201 2023-09-06T20:33:04,707 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(130): Thread=604 is superior to 500 2023-09-06T20:33:04,707 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] backup.TestFullRestore(322): test restore fails on multiple tables that do not exist: command-line 2023-09-06T20:33:04,712 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c04bd9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:04,716 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:04,717 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:38830, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:33:04,719 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:33:04,721 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:53558, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:33:04,734 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:33:04,738 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:33:04,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032384738 (current time:1694032384738). 2023-09-06T20:33:04,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:33:04,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:33:04,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391881c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:04,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:04,749 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:38832, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:33:04,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:33:04,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:33:04,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=89, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:33:04,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 89 2023-09-06T20:33:04,753 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:33:04,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=89 2023-09-06T20:33:04,754 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:33:04,755 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:33:04,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=89 2023-09-06T20:33:05,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=89 2023-09-06T20:33:05,163 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:33:05,163 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281}] 2023-09-06T20:33:05,165 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=90, ppid=89, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 2023-09-06T20:33:05,302 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:33:05,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=90 2023-09-06T20:33:05,317 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:33:05,317 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HRegion(2819): Flushing cf63a85c53279bc94811b50578f31281 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-09-06T20:33:05,323 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/b14bcdd382934779b0fa80ae042378aa is 169, key is trslm:hdfs://localhost:39327/backupUT\x00test-1694032296496/meta:log-roll-map/1694032379113/Put/seqid=0 2023-09-06T20:33:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=89 2023-09-06T20:33:05,727 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/b14bcdd382934779b0fa80ae042378aa 2023-09-06T20:33:05,735 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/8bd33efd4a8e44d1b5b9f6a66a166990 is 452, key is session:backup_1694032372044/session:context/1694032379536/Put/seqid=0 2023-09-06T20:33:05,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=89 2023-09-06T20:33:06,139 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/8bd33efd4a8e44d1b5b9f6a66a166990 2023-09-06T20:33:06,145 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/b14bcdd382934779b0fa80ae042378aa as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/b14bcdd382934779b0fa80ae042378aa 2023-09-06T20:33:06,151 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/b14bcdd382934779b0fa80ae042378aa, entries=4, sequenceid=72, filesize=5.5 K 2023-09-06T20:33:06,151 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/8bd33efd4a8e44d1b5b9f6a66a166990 as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/8bd33efd4a8e44d1b5b9f6a66a166990 2023-09-06T20:33:06,157 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/8bd33efd4a8e44d1b5b9f6a66a166990, entries=2, sequenceid=72, filesize=5.4 K 2023-09-06T20:33:06,157 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for cf63a85c53279bc94811b50578f31281 in 840ms, sequenceid=72, compaction requested=true 2023-09-06T20:33:06,157 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HRegion(2520): Flush status journal for cf63a85c53279bc94811b50578f31281: 2023-09-06T20:33:06,157 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. for snapshot_backup_system completed. 2023-09-06T20:33:06,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:33:06,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:33:06,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/0379b1e1f59e40d2ac2991b2ae6b86ce, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/dd7edbeade334fa7895ff079e0fc3472, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/78eec26cfa0242e48353b19a4a00b7d5, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/faffbcb72f8649bcad705936a3105fd4, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/01fb4a2eb68c4823807811190aa11f8b, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/b14bcdd382934779b0fa80ae042378aa] hfiles 2023-09-06T20:33:06,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (1/6): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/0379b1e1f59e40d2ac2991b2ae6b86ce for snapshot=snapshot_backup_system 2023-09-06T20:33:06,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (2/6): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/dd7edbeade334fa7895ff079e0fc3472 for snapshot=snapshot_backup_system 2023-09-06T20:33:06,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (3/6): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/78eec26cfa0242e48353b19a4a00b7d5 for snapshot=snapshot_backup_system 2023-09-06T20:33:06,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (4/6): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/faffbcb72f8649bcad705936a3105fd4 for snapshot=snapshot_backup_system 2023-09-06T20:33:06,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (5/6): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/01fb4a2eb68c4823807811190aa11f8b for snapshot=snapshot_backup_system 2023-09-06T20:33:06,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (6/6): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/b14bcdd382934779b0fa80ae042378aa for snapshot=snapshot_backup_system 2023-09-06T20:33:06,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/4ced064cdbd34bd6b594c76f049fa4ee, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/baca9a30045a4dabad9b3b78a8fd6c86, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/bcfde09dc46641bebf14d756f321d0af, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/2dc68dbd9f3749a1a2d9d7333fa4771d, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/6da826e7b47e41daa41e268fb4e951b2, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/849e36507ab34f85a508bffbc6e215d3, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/8bd33efd4a8e44d1b5b9f6a66a166990] hfiles 2023-09-06T20:33:06,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (1/7): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/4ced064cdbd34bd6b594c76f049fa4ee for snapshot=snapshot_backup_system 2023-09-06T20:33:06,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (2/7): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/baca9a30045a4dabad9b3b78a8fd6c86 for snapshot=snapshot_backup_system 2023-09-06T20:33:06,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (3/7): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/bcfde09dc46641bebf14d756f321d0af for snapshot=snapshot_backup_system 2023-09-06T20:33:06,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (4/7): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/2dc68dbd9f3749a1a2d9d7333fa4771d for snapshot=snapshot_backup_system 2023-09-06T20:33:06,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (5/7): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/6da826e7b47e41daa41e268fb4e951b2 for snapshot=snapshot_backup_system 2023-09-06T20:33:06,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (6/7): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/849e36507ab34f85a508bffbc6e215d3 for snapshot=snapshot_backup_system 2023-09-06T20:33:06,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (7/7): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/8bd33efd4a8e44d1b5b9f6a66a166990 for snapshot=snapshot_backup_system 2023-09-06T20:33:06,247 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1694032296496' 2023-09-06T20:33:06,566 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:33:06,566 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=90 2023-09-06T20:33:06,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=90 2023-09-06T20:33:06,566 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region cf63a85c53279bc94811b50578f31281 2023-09-06T20:33:06,567 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=90, ppid=89, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 2023-09-06T20:33:06,569 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=90, resume processing ppid=89 2023-09-06T20:33:06,569 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:33:06,569 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=90, ppid=89, state=SUCCESS; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 in 1.4050 sec 2023-09-06T20:33:06,570 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:33:06,570 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:33:06,570 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:33:06,571 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:33:06,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=89 2023-09-06T20:33:06,984 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:33:06,996 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:33:06,996 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:33:06,997 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:33:06,998 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 89 2023-09-06T20:33:06,999 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=89, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2470 sec 2023-09-06T20:33:08,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=89 2023-09-06T20:33:08,860 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(115): Backup backup_1694032384713 started at 1694032388860. 2023-09-06T20:33:08,861 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(119): Backup session backup_1694032384713 has been started. 2023-09-06T20:33:08,866 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-09-06T20:33:08,867 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-09-06T20:33:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-09-06T20:33:08,867 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-09-06T20:33:08,867 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-09-06T20:33:08,868 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-09-06T20:33:08,868 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-09-06T20:33:08,869 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:33:08,870 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:08,870 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:33:08,870 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:33:08,870 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-09-06T20:33:08,871 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-09-06T20:33:08,871 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-09-06T20:33:08,871 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:33:08,871 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-09-06T20:33:08,871 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-09-06T20:33:08,871 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-09-06T20:33:08,871 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-09-06T20:33:08,871 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-09-06T20:33:08,872 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-09-06T20:33:08,872 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-09-06T20:33:08,872 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,36961,1694032287907' joining acquired barrier for procedure (rolllog) in zk 2023-09-06T20:33:08,874 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-09-06T20:33:08,874 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:08,874 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:33:08,875 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:33:08,875 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-09-06T20:33:08,875 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:33:08,875 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:33:08,875 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:33:08,876 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:08,876 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:33:08,876 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,36961,1694032287907' joining acquired barrier for procedure 'rolllog' on coordinator 2023-09-06T20:33:08,876 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-09-06T20:33:08,876 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@41c857c[Count = 0] remaining members to acquire global barrier 2023-09-06T20:33:08,876 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-09-06T20:33:08,878 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-09-06T20:33:08,878 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-09-06T20:33:08,878 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-09-06T20:33:08,878 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-09-06T20:33:08,878 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-09-06T20:33:08,878 DEBUG [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:08,878 INFO [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1694032374233 highest: 1694032374233 on jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:08,878 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta:.meta(num 1694032374214) roll requested 2023-09-06T20:33:08,890 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032388878.meta, exclude list is [], retry=0 2023-09-06T20:33:08,894 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK] 2023-09-06T20:33:08,895 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032374214.meta with entries=12, filesize=3.07 KB; new WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032388878.meta 2023-09-06T20:33:08,895 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK]] 2023-09-06T20:33:08,895 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032374214.meta is not closed yet, will try archiving it next time 2023-09-06T20:33:08,895 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36961%2C1694032287907:(num 1694032374233) roll requested 2023-09-06T20:33:08,897 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032374214.meta not finished, retry = 0 2023-09-06T20:33:08,907 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032388895, exclude list is [], retry=0 2023-09-06T20:33:08,909 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK] 2023-09-06T20:33:08,911 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032374233 with entries=15, filesize=4.76 KB; new WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032388895 2023-09-06T20:33:08,911 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK]] 2023-09-06T20:33:08,911 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032374233 is not closed yet, will try archiving it next time 2023-09-06T20:33:08,911 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032361884 to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/oldWALs/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032361884 2023-09-06T20:33:08,918 DEBUG [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-09-06T20:33:08,918 INFO [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1694032388895 on jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:08,926 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-09-06T20:33:08,926 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,36961,1694032287907' in zk 2023-09-06T20:33:08,929 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-09-06T20:33:08,930 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:33:08,930 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:08,930 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-09-06T20:33:08,930 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:33:08,930 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:33:08,931 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:33:08,931 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:33:08,931 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:33:08,932 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:08,932 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:33:08,932 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:33:08,932 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:08,933 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,36961,1694032287907': 2023-09-06T20:33:08,933 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,36961,1694032287907' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-09-06T20:33:08,933 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-09-06T20:33:08,933 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-09-06T20:33:08,933 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-09-06T20:33:08,933 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-09-06T20:33:08,933 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-09-06T20:33:08,935 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-09-06T20:33:08,935 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-09-06T20:33:08,935 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:33:08,935 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:33:08,935 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:33:08,935 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:33:08,935 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:33:08,935 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:33:08,936 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:33:08,936 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:33:08,936 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:33:08,936 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:33:08,937 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:08,937 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:33:08,937 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:33:08,937 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:08,940 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:33:08,941 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:33:08,941 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:33:08,941 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-09-06T20:33:08,941 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-09-06T20:33:08,941 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:33:08,941 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:33:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-09-06T20:33:09,045 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1694032389044_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } 2023-09-06T20:33:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032389045 (current time:1694032389045). 2023-09-06T20:33:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:33:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1694032389044_ns2_test-16940322964961 VERSION not specified, setting to 2 2023-09-06T20:33:09,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1331d223, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:09,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:09,054 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:38836, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:33:09,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:33:09,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:33:09,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=91, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1694032389044_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } 2023-09-06T20:33:09,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1694032389044_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 }, snapshot procedure id = 91 2023-09-06T20:33:09,057 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1694032389044_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:33:09,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=91 2023-09-06T20:33:09,058 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1694032389044_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:33:09,059 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1694032389044_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:33:09,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=91 2023-09-06T20:33:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=91 2023-09-06T20:33:09,468 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1694032389044_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:33:09,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE; SnapshotRegionProcedure cd8331e491339ba2021603869fc044dc}] 2023-09-06T20:33:09,470 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=92, ppid=91, state=RUNNABLE; SnapshotRegionProcedure cd8331e491339ba2021603869fc044dc 2023-09-06T20:33:09,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=92 2023-09-06T20:33:09,622 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc. 2023-09-06T20:33:09,622 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] regionserver.HRegion(2520): Flush status journal for cd8331e491339ba2021603869fc044dc: 2023-09-06T20:33:09,622 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc. for snapshot_1694032389044_ns2_test-16940322964961 completed. 2023-09-06T20:33:09,622 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc.' region-info for snapshot=snapshot_1694032389044_ns2_test-16940322964961 2023-09-06T20:33:09,622 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:33:09,622 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc/f/5f020f30827940489104cc2b9277b6d8_SeqId_4_] hfiles 2023-09-06T20:33:09,622 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc/f/5f020f30827940489104cc2b9277b6d8_SeqId_4_ for snapshot=snapshot_1694032389044_ns2_test-16940322964961 2023-09-06T20:33:09,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=91 2023-09-06T20:33:10,028 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc. 2023-09-06T20:33:10,028 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=92 2023-09-06T20:33:10,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=92 2023-09-06T20:33:10,029 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1694032389044_ns2_test-16940322964961 on region cd8331e491339ba2021603869fc044dc 2023-09-06T20:33:10,030 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=92, ppid=91, state=RUNNABLE; SnapshotRegionProcedure cd8331e491339ba2021603869fc044dc 2023-09-06T20:33:10,032 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=92, resume processing ppid=91 2023-09-06T20:33:10,032 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=92, ppid=91, state=SUCCESS; SnapshotRegionProcedure cd8331e491339ba2021603869fc044dc in 562 msec 2023-09-06T20:33:10,032 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1694032389044_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:33:10,033 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1694032389044_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:33:10,033 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1694032389044_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:33:10,033 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1694032389044_ns2_test-16940322964961 2023-09-06T20:33:10,034 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032389044_ns2_test-16940322964961 2023-09-06T20:33:10,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=91 2023-09-06T20:33:10,443 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1694032389044_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:33:10,448 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1694032389044_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:33:10,448 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032389044_ns2_test-16940322964961 to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032389044_ns2_test-16940322964961 2023-09-06T20:33:10,449 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1694032389044_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:33:10,449 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1694032389044_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 }, snapshot procedure id = 91 2023-09-06T20:33:10,450 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=91, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1694032389044_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } in 1.3940 sec 2023-09-06T20:33:11,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=91 2023-09-06T20:33:11,165 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1694032391164_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } 2023-09-06T20:33:11,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032391165 (current time:1694032391165). 2023-09-06T20:33:11,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:33:11,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1694032391164_ns3_test-16940322964962 VERSION not specified, setting to 2 2023-09-06T20:33:11,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64c97de0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:11,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:11,174 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:35472, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:33:11,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:33:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:33:11,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=93, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1694032391164_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } 2023-09-06T20:33:11,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1694032391164_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 }, snapshot procedure id = 93 2023-09-06T20:33:11,177 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1694032391164_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:33:11,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=93 2023-09-06T20:33:11,178 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1694032391164_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:33:11,179 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1694032391164_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:33:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=93 2023-09-06T20:33:11,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=93 2023-09-06T20:33:11,585 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1694032391164_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:33:11,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE; SnapshotRegionProcedure 685fcf26b1065c7882673d26a6d8d168}] 2023-09-06T20:33:11,587 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=94, ppid=93, state=RUNNABLE; SnapshotRegionProcedure 685fcf26b1065c7882673d26a6d8d168 2023-09-06T20:33:11,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=94 2023-09-06T20:33:11,739 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168. 2023-09-06T20:33:11,740 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] regionserver.HRegion(2520): Flush status journal for 685fcf26b1065c7882673d26a6d8d168: 2023-09-06T20:33:11,740 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168. for snapshot_1694032391164_ns3_test-16940322964962 completed. 2023-09-06T20:33:11,740 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168.' region-info for snapshot=snapshot_1694032391164_ns3_test-16940322964962 2023-09-06T20:33:11,740 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:33:11,740 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-09-06T20:33:11,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=93 2023-09-06T20:33:12,146 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168. 2023-09-06T20:33:12,146 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=94 2023-09-06T20:33:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=94 2023-09-06T20:33:12,146 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1694032391164_ns3_test-16940322964962 on region 685fcf26b1065c7882673d26a6d8d168 2023-09-06T20:33:12,148 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=94, ppid=93, state=RUNNABLE; SnapshotRegionProcedure 685fcf26b1065c7882673d26a6d8d168 2023-09-06T20:33:12,150 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=94, resume processing ppid=93 2023-09-06T20:33:12,150 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=94, ppid=93, state=SUCCESS; SnapshotRegionProcedure 685fcf26b1065c7882673d26a6d8d168 in 563 msec 2023-09-06T20:33:12,150 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1694032391164_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:33:12,150 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1694032391164_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:33:12,151 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1694032391164_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:33:12,151 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1694032391164_ns3_test-16940322964962 2023-09-06T20:33:12,151 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032391164_ns3_test-16940322964962 2023-09-06T20:33:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=93 2023-09-06T20:33:12,560 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1694032391164_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:33:12,565 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1694032391164_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:33:12,565 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032391164_ns3_test-16940322964962 to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032391164_ns3_test-16940322964962 2023-09-06T20:33:12,567 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1694032391164_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:33:12,567 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1694032391164_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 }, snapshot procedure id = 93 2023-09-06T20:33:12,568 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=93, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1694032391164_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } in 1.3920 sec 2023-09-06T20:33:13,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=93 2023-09-06T20:33:13,282 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1694032384713 2023-09-06T20:33:13,282 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-09-06T20:33:13,282 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-09-06T20:33:13,282 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1694032384713_ns3:test-16940322964962 2023-09-06T20:33:13,282 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1694032391164_ns3_test-16940322964962 to hdfs://localhost:39327/backupUT/backup_1694032384713/ns3/test-16940322964962/ 2023-09-06T20:33:13,282 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-09-06T20:33:13,305 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:33:13,305 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37419,1694032284546_1691994294_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:39327/backupUT/backup_1694032384713/ns3/test-16940322964962, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:39327/backupUT/backup_1694032384713/ns3/test-16940322964962/.hbase-snapshot/.tmp/snapshot_1694032391164_ns3_test-16940322964962 2023-09-06T20:33:13,305 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1, snapshotDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032391164_ns3_test-16940322964962. 2023-09-06T20:33:13,313 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032391164_ns3_test-16940322964962 to hdfs://localhost:39327/backupUT/backup_1694032384713/ns3/test-16940322964962/.hbase-snapshot/.tmp/snapshot_1694032391164_ns3_test-16940322964962 2023-09-06T20:33:13,728 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:13,728 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:13,729 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:13,729 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:14,727 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop-3199444996374623032.jar 2023-09-06T20:33:14,728 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:14,728 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:14,729 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:14,729 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:14,730 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:14,731 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:14,731 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:14,732 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:14,732 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:14,733 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:33:14,734 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:33:14,735 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:33:14,735 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:33:14,736 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:33:14,737 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:33:14,737 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:33:14,738 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:33:14,739 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:33:14,739 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:33:14,740 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:33:14,740 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:33:14,741 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:33:14,742 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:33:14,742 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:33:14,743 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:33:14,743 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:33:14,743 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:33:14,744 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:33:14,790 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1694032391164_ns3_test-16940322964962' hfile list 2023-09-06T20:33:14,890 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032394849/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:33:14,890 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032394849/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:33:15,919 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-09-06T20:33:15,920 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-09-06T20:33:15,924 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1694032391164_ns3_test-16940322964962 2023-09-06T20:33:15,924 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1694032391164_ns3_test-16940322964962 finished. 2023-09-06T20:33:15,924 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1694032384713_ns2:test-16940322964961 2023-09-06T20:33:15,924 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1694032389044_ns2_test-16940322964961 to hdfs://localhost:39327/backupUT/backup_1694032384713/ns2/test-16940322964961/ 2023-09-06T20:33:15,924 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-09-06T20:33:15,947 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:33:15,947 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37419,1694032284546_-912479241_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:39327/backupUT/backup_1694032384713/ns2/test-16940322964961, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:39327/backupUT/backup_1694032384713/ns2/test-16940322964961/.hbase-snapshot/.tmp/snapshot_1694032389044_ns2_test-16940322964961 2023-09-06T20:33:15,947 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1, snapshotDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032389044_ns2_test-16940322964961. 2023-09-06T20:33:15,953 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032389044_ns2_test-16940322964961 to hdfs://localhost:39327/backupUT/backup_1694032384713/ns2/test-16940322964961/.hbase-snapshot/.tmp/snapshot_1694032389044_ns2_test-16940322964961 2023-09-06T20:33:16,367 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:16,368 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:16,368 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:16,368 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:17,390 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop-7527712290695767675.jar 2023-09-06T20:33:17,391 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:17,391 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:17,392 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:17,392 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:17,392 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:17,393 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:17,393 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:17,393 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:17,393 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:17,394 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:33:17,395 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:33:17,395 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:33:17,396 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:33:17,396 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:33:17,396 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:33:17,397 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:33:17,397 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:33:17,397 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:33:17,398 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:33:17,398 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:33:17,398 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:33:17,399 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:33:17,399 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:33:17,400 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:33:17,400 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:33:17,401 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:33:17,401 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:33:17,402 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:33:17,467 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1694032389044_ns2_test-16940322964961' hfile list 2023-09-06T20:33:17,469 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-09-06T20:33:17,537 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032397504/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:33:17,537 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032397504/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:33:17,591 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-09-06T20:33:17,615 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc/f/5f020f30827940489104cc2b9277b6d8_SeqId_4_ output=hdfs://localhost:39327/backupUT/backup_1694032384713/ns2/test-16940322964961/archive/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc/f/5f020f30827940489104cc2b9277b6d8_SeqId_4_ 2023-09-06T20:33:17,615 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 8.320M/sec 2023-09-06T20:33:18,569 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-09-06T20:33:18,569 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-09-06T20:33:18,575 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1694032389044_ns2_test-16940322964961 2023-09-06T20:33:18,575 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1694032389044_ns2_test-16940322964961 finished. 2023-09-06T20:33:18,575 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1014): ns3:test-16940322964962 2023-09-06T20:33:18,575 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1014): ns2:test-16940322964961 2023-09-06T20:33:18,581 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1694032384713 2023-09-06T20:33:18,581 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-09-06T20:33:18,986 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:39327/backupUT/backup_1694032384713/.backup.manifest 2023-09-06T20:33:18,986 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1694032384713 2023-09-06T20:33:18,986 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-09-06T20:33:18,991 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:39327/backupUT/backup_1694032384713/.backup.manifest 2023-09-06T20:33:18,991 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(358): Backup backup_1694032384713 finished: type=FULL,tablelist=ns3:test-16940322964962;ns2:test-16940322964961,targetRootDir=hdfs://localhost:39327/backupUT,startts=1694032388860,completets=1694032398581,bytescopied=0 2023-09-06T20:33:18,991 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-09-06T20:33:18,991 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1694032391164_ns3_test-16940322964962 2023-09-06T20:33:18,992 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1694032391164_ns3_test-16940322964962" 2023-09-06T20:33:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1694032391164_ns3_test-16940322964962 2023-09-06T20:33:18,996 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1694032391164_ns3_test-16940322964962 for backup backup_1694032384713 succeeded. 2023-09-06T20:33:18,996 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1694032389044_ns2_test-16940322964961 2023-09-06T20:33:18,996 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1694032389044_ns2_test-16940322964961" 2023-09-06T20:33:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1694032389044_ns2_test-16940322964961 2023-09-06T20:33:18,999 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1694032389044_ns2_test-16940322964961 for backup backup_1694032384713 succeeded. 2023-09-06T20:33:19,000 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:33:19,006 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:33:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:33:19,008 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:33:19,010 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:33:19,016 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(376): Backup backup_1694032384713 completed. 2023-09-06T20:33:19,016 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client You have to specify either set name or table list to restore Usage: hbase restore [options] backup_path Path to a backup destination root backup_id Backup image ID to restore table(s) Comma-separated list of tables to restore Options: -c Check restore sequence and dependencies only (does not execute the command) -d Enable debug loggings -h,--help Show usage -m A comma separated list of target tables. If specified, each table in must have a mapping -o Overwrite data if any of the restore target tables exists -q Yarn queue name to run backup restore command on -s Backup set to restore, mutually exclusive with -t (table list) -t Table name list, comma-separated. To enable backup, in hbase-site.xml, set: hbase.backup.enable=true hbase.master.logcleaner.plugins=YOUR_PLUGINS,org.apache.hadoop.hbase.backup.master.BackupLogCleaner hbase.procedure.master.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager hbase.procedure.regionserver.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager hbase.coprocessor.region.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.BackupObserver and restart the cluster For more information please see http://hbase.apache.org/book.html#backuprestore 2023-09-06T20:33:19,042 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultipleDNECommand Thread=597 (was 604), OpenFileDescriptor=682 (was 694), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=143 (was 151), ProcessCount=167 (was 167), AvailableMemoryMB=3103 (was 3201) 2023-09-06T20:33:19,042 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(130): Thread=597 is superior to 500 2023-09-06T20:33:19,056 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultiple Thread=597, OpenFileDescriptor=682, MaxFileDescriptor=60000, SystemLoadAverage=143, ProcessCount=167, AvailableMemoryMB=3102 2023-09-06T20:33:19,057 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(130): Thread=597 is superior to 500 2023-09-06T20:33:19,057 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] backup.TestFullRestore(120): create full backup image on multiple tables 2023-09-06T20:33:19,062 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@573c1f7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:19,065 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:19,066 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:35480, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:33:19,068 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:33:19,070 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:57148, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:33:19,081 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:33:19,086 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:33:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032399086 (current time:1694032399086). 2023-09-06T20:33:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:33:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:33:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25357e44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:19,099 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:35486, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:33:19,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:33:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:33:19,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=95, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:33:19,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 95 2023-09-06T20:33:19,103 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:33:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=95 2023-09-06T20:33:19,103 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:33:19,105 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:33:19,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=95 2023-09-06T20:33:19,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=95 2023-09-06T20:33:19,511 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:33:19,511 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281}] 2023-09-06T20:33:19,513 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=96, ppid=95, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 2023-09-06T20:33:19,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=96 2023-09-06T20:33:19,665 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:33:19,665 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HRegion(2819): Flushing cf63a85c53279bc94811b50578f31281 2/2 column families, dataSize=1.50 KB heapSize=2.73 KB 2023-09-06T20:33:19,669 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/b7aca3aa47f24516bd2686bfd6b9193d is 171, key is trslm:hdfs://localhost:39327/backupUT\x00ns2:test-16940322964961/meta:log-roll-map/1694032398577/Put/seqid=0 2023-09-06T20:33:19,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=95 2023-09-06T20:33:20,073 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=717 B at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/b7aca3aa47f24516bd2686bfd6b9193d 2023-09-06T20:33:20,082 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/13357027918440549abec030e3035318 is 524, key is session:backup_1694032384713/session:context/1694032399009/Put/seqid=0 2023-09-06T20:33:20,086 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=820 B at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/13357027918440549abec030e3035318 2023-09-06T20:33:20,092 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/b7aca3aa47f24516bd2686bfd6b9193d as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/b7aca3aa47f24516bd2686bfd6b9193d 2023-09-06T20:33:20,096 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/b7aca3aa47f24516bd2686bfd6b9193d, entries=6, sequenceid=83, filesize=5.8 K 2023-09-06T20:33:20,097 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/13357027918440549abec030e3035318 as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/13357027918440549abec030e3035318 2023-09-06T20:33:20,102 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/13357027918440549abec030e3035318, entries=2, sequenceid=83, filesize=5.5 K 2023-09-06T20:33:20,103 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HRegion(3022): Finished flush of dataSize ~1.50 KB/1537, heapSize ~2.70 KB/2760, currentSize=0 B/0 for cf63a85c53279bc94811b50578f31281 in 437ms, sequenceid=83, compaction requested=true 2023-09-06T20:33:20,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HRegion(2520): Flush status journal for cf63a85c53279bc94811b50578f31281: 2023-09-06T20:33:20,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. for snapshot_backup_system completed. 2023-09-06T20:33:20,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:33:20,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:33:20,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/0379b1e1f59e40d2ac2991b2ae6b86ce, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/dd7edbeade334fa7895ff079e0fc3472, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/78eec26cfa0242e48353b19a4a00b7d5, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/faffbcb72f8649bcad705936a3105fd4, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/01fb4a2eb68c4823807811190aa11f8b, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/b14bcdd382934779b0fa80ae042378aa, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/b7aca3aa47f24516bd2686bfd6b9193d] hfiles 2023-09-06T20:33:20,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (1/7): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/0379b1e1f59e40d2ac2991b2ae6b86ce for snapshot=snapshot_backup_system 2023-09-06T20:33:20,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (2/7): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/dd7edbeade334fa7895ff079e0fc3472 for snapshot=snapshot_backup_system 2023-09-06T20:33:20,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (3/7): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/78eec26cfa0242e48353b19a4a00b7d5 for snapshot=snapshot_backup_system 2023-09-06T20:33:20,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (4/7): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/faffbcb72f8649bcad705936a3105fd4 for snapshot=snapshot_backup_system 2023-09-06T20:33:20,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (5/7): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/01fb4a2eb68c4823807811190aa11f8b for snapshot=snapshot_backup_system 2023-09-06T20:33:20,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (6/7): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/b14bcdd382934779b0fa80ae042378aa for snapshot=snapshot_backup_system 2023-09-06T20:33:20,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (7/7): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/b7aca3aa47f24516bd2686bfd6b9193d for snapshot=snapshot_backup_system 2023-09-06T20:33:20,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/4ced064cdbd34bd6b594c76f049fa4ee, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/baca9a30045a4dabad9b3b78a8fd6c86, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/bcfde09dc46641bebf14d756f321d0af, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/2dc68dbd9f3749a1a2d9d7333fa4771d, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/6da826e7b47e41daa41e268fb4e951b2, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/849e36507ab34f85a508bffbc6e215d3, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/8bd33efd4a8e44d1b5b9f6a66a166990, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/13357027918440549abec030e3035318] hfiles 2023-09-06T20:33:20,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (1/8): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/4ced064cdbd34bd6b594c76f049fa4ee for snapshot=snapshot_backup_system 2023-09-06T20:33:20,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (2/8): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/baca9a30045a4dabad9b3b78a8fd6c86 for snapshot=snapshot_backup_system 2023-09-06T20:33:20,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (3/8): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/bcfde09dc46641bebf14d756f321d0af for snapshot=snapshot_backup_system 2023-09-06T20:33:20,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (4/8): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/2dc68dbd9f3749a1a2d9d7333fa4771d for snapshot=snapshot_backup_system 2023-09-06T20:33:20,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (5/8): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/6da826e7b47e41daa41e268fb4e951b2 for snapshot=snapshot_backup_system 2023-09-06T20:33:20,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (6/8): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/849e36507ab34f85a508bffbc6e215d3 for snapshot=snapshot_backup_system 2023-09-06T20:33:20,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (7/8): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/8bd33efd4a8e44d1b5b9f6a66a166990 for snapshot=snapshot_backup_system 2023-09-06T20:33:20,103 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (8/8): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/13357027918440549abec030e3035318 for snapshot=snapshot_backup_system 2023-09-06T20:33:20,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=95 2023-09-06T20:33:20,509 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:33:20,509 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=96 2023-09-06T20:33:20,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=96 2023-09-06T20:33:20,509 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region cf63a85c53279bc94811b50578f31281 2023-09-06T20:33:20,512 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=96, ppid=95, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 2023-09-06T20:33:20,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=96, resume processing ppid=95 2023-09-06T20:33:20,514 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:33:20,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=96, ppid=95, state=SUCCESS; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 in 1.0020 sec 2023-09-06T20:33:20,515 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:33:20,515 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:33:20,515 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:33:20,516 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:33:20,927 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:33:20,939 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:33:20,939 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:33:20,940 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:33:20,940 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 95 2023-09-06T20:33:20,941 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=95, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8400 sec 2023-09-06T20:33:21,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=95 2023-09-06T20:33:21,208 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(115): Backup backup_1694032399062 started at 1694032401208. 2023-09-06T20:33:21,210 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(119): Backup session backup_1694032399062 has been started. 2023-09-06T20:33:21,215 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-09-06T20:33:21,215 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-09-06T20:33:21,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-09-06T20:33:21,216 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-09-06T20:33:21,216 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-09-06T20:33:21,217 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-09-06T20:33:21,217 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-09-06T20:33:21,217 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:33:21,222 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:21,222 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:33:21,222 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:33:21,222 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-09-06T20:33:21,222 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-09-06T20:33:21,223 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-09-06T20:33:21,223 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:33:21,223 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-09-06T20:33:21,223 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-09-06T20:33:21,223 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-09-06T20:33:21,223 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-09-06T20:33:21,223 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-09-06T20:33:21,224 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-09-06T20:33:21,224 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-09-06T20:33:21,224 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,36961,1694032287907' joining acquired barrier for procedure (rolllog) in zk 2023-09-06T20:33:21,226 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-09-06T20:33:21,226 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:21,226 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:33:21,226 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:33:21,226 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-09-06T20:33:21,226 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:33:21,227 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:33:21,227 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:33:21,227 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:21,227 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:33:21,227 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,36961,1694032287907' joining acquired barrier for procedure 'rolllog' on coordinator 2023-09-06T20:33:21,227 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@76fe9f60[Count = 0] remaining members to acquire global barrier 2023-09-06T20:33:21,228 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-09-06T20:33:21,228 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-09-06T20:33:21,230 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-09-06T20:33:21,230 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-09-06T20:33:21,230 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-09-06T20:33:21,230 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-09-06T20:33:21,230 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-09-06T20:33:21,230 DEBUG [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:21,230 INFO [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1694032388895 highest: 1694032388895 on jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:21,230 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta:.meta(num 1694032388878) roll requested 2023-09-06T20:33:21,241 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032401230.meta, exclude list is [], retry=0 2023-09-06T20:33:21,247 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK] 2023-09-06T20:33:21,248 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032388878.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032401230.meta 2023-09-06T20:33:21,248 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK]] 2023-09-06T20:33:21,248 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032388878.meta is not closed yet, will try archiving it next time 2023-09-06T20:33:21,248 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36961%2C1694032287907:(num 1694032388895) roll requested 2023-09-06T20:33:21,251 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032388878.meta not finished, retry = 0 2023-09-06T20:33:21,260 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032401249, exclude list is [], retry=0 2023-09-06T20:33:21,264 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK] 2023-09-06T20:33:21,265 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032388895 with entries=10, filesize=2.67 KB; new WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032401249 2023-09-06T20:33:21,266 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK]] 2023-09-06T20:33:21,266 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032388895 is not closed yet, will try archiving it next time 2023-09-06T20:33:21,267 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032388895 not finished, retry = 0 2023-09-06T20:33:21,270 DEBUG [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-09-06T20:33:21,270 INFO [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1694032401249 on jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:21,279 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-09-06T20:33:21,279 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,36961,1694032287907' in zk 2023-09-06T20:33:21,282 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-09-06T20:33:21,282 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:33:21,282 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:21,282 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-09-06T20:33:21,283 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:33:21,283 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:33:21,283 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:33:21,284 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:33:21,284 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:33:21,284 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:21,284 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:33:21,285 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:33:21,285 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:21,285 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,36961,1694032287907': 2023-09-06T20:33:21,286 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,36961,1694032287907' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-09-06T20:33:21,286 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-09-06T20:33:21,286 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-09-06T20:33:21,286 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-09-06T20:33:21,286 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-09-06T20:33:21,286 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-09-06T20:33:21,287 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-09-06T20:33:21,287 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-09-06T20:33:21,287 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:33:21,288 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:33:21,288 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:33:21,288 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:33:21,288 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:33:21,288 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:33:21,288 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:33:21,288 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:33:21,289 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:33:21,289 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:33:21,289 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:21,290 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:33:21,290 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:33:21,290 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:21,294 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:33:21,294 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:33:21,294 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-09-06T20:33:21,294 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:33:21,294 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-09-06T20:33:21,295 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:33:21,295 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:33:21,352 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032388878.meta to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/oldWALs/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032388878.meta 2023-09-06T20:33:21,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-09-06T20:33:21,399 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1694032401399_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } 2023-09-06T20:33:21,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032401399 (current time:1694032401399). 2023-09-06T20:33:21,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:33:21,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1694032401399_ns2_test-16940322964961 VERSION not specified, setting to 2 2023-09-06T20:33:21,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a85a86b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:21,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:21,409 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48144, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:33:21,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:33:21,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:33:21,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=97, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1694032401399_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } 2023-09-06T20:33:21,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1694032401399_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 }, snapshot procedure id = 97 2023-09-06T20:33:21,413 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1694032401399_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:33:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=97 2023-09-06T20:33:21,414 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1694032401399_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:33:21,415 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1694032401399_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:33:21,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=97 2023-09-06T20:33:21,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=97 2023-09-06T20:33:21,820 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1694032401399_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:33:21,821 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE; SnapshotRegionProcedure cd8331e491339ba2021603869fc044dc}] 2023-09-06T20:33:21,822 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=98, ppid=97, state=RUNNABLE; SnapshotRegionProcedure cd8331e491339ba2021603869fc044dc 2023-09-06T20:33:21,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=98 2023-09-06T20:33:21,974 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc. 2023-09-06T20:33:21,974 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] regionserver.HRegion(2520): Flush status journal for cd8331e491339ba2021603869fc044dc: 2023-09-06T20:33:21,974 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc. for snapshot_1694032401399_ns2_test-16940322964961 completed. 2023-09-06T20:33:21,974 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc.' region-info for snapshot=snapshot_1694032401399_ns2_test-16940322964961 2023-09-06T20:33:21,974 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:33:21,974 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc/f/5f020f30827940489104cc2b9277b6d8_SeqId_4_] hfiles 2023-09-06T20:33:21,974 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc/f/5f020f30827940489104cc2b9277b6d8_SeqId_4_ for snapshot=snapshot_1694032401399_ns2_test-16940322964961 2023-09-06T20:33:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=97 2023-09-06T20:33:22,380 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc. 2023-09-06T20:33:22,380 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=98 2023-09-06T20:33:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=98 2023-09-06T20:33:22,381 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1694032401399_ns2_test-16940322964961 on region cd8331e491339ba2021603869fc044dc 2023-09-06T20:33:22,382 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=98, ppid=97, state=RUNNABLE; SnapshotRegionProcedure cd8331e491339ba2021603869fc044dc 2023-09-06T20:33:22,384 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=98, resume processing ppid=97 2023-09-06T20:33:22,384 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1694032401399_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:33:22,384 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=98, ppid=97, state=SUCCESS; SnapshotRegionProcedure cd8331e491339ba2021603869fc044dc in 562 msec 2023-09-06T20:33:22,384 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1694032401399_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:33:22,385 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1694032401399_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:33:22,385 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1694032401399_ns2_test-16940322964961 2023-09-06T20:33:22,385 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032401399_ns2_test-16940322964961 2023-09-06T20:33:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=97 2023-09-06T20:33:22,794 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1694032401399_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:33:22,799 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1694032401399_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:33:22,800 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032401399_ns2_test-16940322964961 to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032401399_ns2_test-16940322964961 2023-09-06T20:33:22,801 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1694032401399_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:33:22,801 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1694032401399_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 }, snapshot procedure id = 97 2023-09-06T20:33:22,802 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=97, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1694032401399_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } in 1.3900 sec 2023-09-06T20:33:23,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=97 2023-09-06T20:33:23,518 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1694032403518_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } 2023-09-06T20:33:23,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032403518 (current time:1694032403518). 2023-09-06T20:33:23,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:33:23,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1694032403518_ns3_test-16940322964962 VERSION not specified, setting to 2 2023-09-06T20:33:23,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2443d991, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:23,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:23,527 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48160, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:33:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:33:23,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:33:23,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=99, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1694032403518_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } 2023-09-06T20:33:23,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1694032403518_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 }, snapshot procedure id = 99 2023-09-06T20:33:23,530 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1694032403518_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:33:23,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=99 2023-09-06T20:33:23,531 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1694032403518_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:33:23,532 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1694032403518_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:33:23,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=99 2023-09-06T20:33:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=99 2023-09-06T20:33:23,938 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1694032403518_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:33:23,938 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE; SnapshotRegionProcedure 685fcf26b1065c7882673d26a6d8d168}] 2023-09-06T20:33:23,940 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=100, ppid=99, state=RUNNABLE; SnapshotRegionProcedure 685fcf26b1065c7882673d26a6d8d168 2023-09-06T20:33:24,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=100 2023-09-06T20:33:24,091 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168. 2023-09-06T20:33:24,092 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] regionserver.HRegion(2520): Flush status journal for 685fcf26b1065c7882673d26a6d8d168: 2023-09-06T20:33:24,092 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168. for snapshot_1694032403518_ns3_test-16940322964962 completed. 2023-09-06T20:33:24,092 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168.' region-info for snapshot=snapshot_1694032403518_ns3_test-16940322964962 2023-09-06T20:33:24,092 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:33:24,092 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-09-06T20:33:24,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=99 2023-09-06T20:33:24,497 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168. 2023-09-06T20:33:24,497 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=100 2023-09-06T20:33:24,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=100 2023-09-06T20:33:24,498 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1694032403518_ns3_test-16940322964962 on region 685fcf26b1065c7882673d26a6d8d168 2023-09-06T20:33:24,499 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=100, ppid=99, state=RUNNABLE; SnapshotRegionProcedure 685fcf26b1065c7882673d26a6d8d168 2023-09-06T20:33:24,501 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=100, resume processing ppid=99 2023-09-06T20:33:24,501 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=100, ppid=99, state=SUCCESS; SnapshotRegionProcedure 685fcf26b1065c7882673d26a6d8d168 in 562 msec 2023-09-06T20:33:24,501 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1694032403518_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:33:24,502 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1694032403518_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:33:24,502 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1694032403518_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:33:24,502 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1694032403518_ns3_test-16940322964962 2023-09-06T20:33:24,503 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032403518_ns3_test-16940322964962 2023-09-06T20:33:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=99 2023-09-06T20:33:24,912 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1694032403518_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:33:24,917 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1694032403518_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:33:24,917 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032403518_ns3_test-16940322964962 to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032403518_ns3_test-16940322964962 2023-09-06T20:33:24,918 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1694032403518_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:33:24,919 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1694032403518_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 }, snapshot procedure id = 99 2023-09-06T20:33:24,919 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=99, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1694032403518_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } in 1.3910 sec 2023-09-06T20:33:25,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=99 2023-09-06T20:33:25,635 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1694032399062 2023-09-06T20:33:25,635 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-09-06T20:33:25,635 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-09-06T20:33:25,635 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1694032399062_ns3:test-16940322964962 2023-09-06T20:33:25,636 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1694032403518_ns3_test-16940322964962 to hdfs://localhost:39327/backupUT/backup_1694032399062/ns3/test-16940322964962/ 2023-09-06T20:33:25,636 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-09-06T20:33:25,659 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:33:25,659 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37419,1694032284546_-607774646_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:39327/backupUT/backup_1694032399062/ns3/test-16940322964962, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:39327/backupUT/backup_1694032399062/ns3/test-16940322964962/.hbase-snapshot/.tmp/snapshot_1694032403518_ns3_test-16940322964962 2023-09-06T20:33:25,659 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1, snapshotDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032403518_ns3_test-16940322964962. 2023-09-06T20:33:25,664 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032403518_ns3_test-16940322964962 to hdfs://localhost:39327/backupUT/backup_1694032399062/ns3/test-16940322964962/.hbase-snapshot/.tmp/snapshot_1694032403518_ns3_test-16940322964962 2023-09-06T20:33:26,082 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:26,082 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:26,083 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:26,083 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:27,055 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop-5938883437823207871.jar 2023-09-06T20:33:27,055 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:27,056 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:27,056 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:27,056 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:27,057 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:27,057 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:27,057 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:27,058 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:27,058 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:27,059 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:33:27,059 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:33:27,060 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:33:27,060 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:33:27,060 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:33:27,061 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:33:27,061 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:33:27,061 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:33:27,062 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:33:27,062 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:33:27,063 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:33:27,063 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:33:27,064 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:33:27,064 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:33:27,065 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:33:27,065 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:33:27,065 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:33:27,066 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:33:27,067 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:33:27,108 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1694032403518_ns3_test-16940322964962' hfile list 2023-09-06T20:33:27,177 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032407144/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:33:27,177 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032407144/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:33:28,205 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-09-06T20:33:28,207 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-09-06T20:33:28,211 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1694032403518_ns3_test-16940322964962 2023-09-06T20:33:28,211 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1694032403518_ns3_test-16940322964962 finished. 2023-09-06T20:33:28,211 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1694032399062_ns2:test-16940322964961 2023-09-06T20:33:28,211 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1694032401399_ns2_test-16940322964961 to hdfs://localhost:39327/backupUT/backup_1694032399062/ns2/test-16940322964961/ 2023-09-06T20:33:28,211 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-09-06T20:33:28,234 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:33:28,234 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37419,1694032284546_2040418750_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:39327/backupUT/backup_1694032399062/ns2/test-16940322964961, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:39327/backupUT/backup_1694032399062/ns2/test-16940322964961/.hbase-snapshot/.tmp/snapshot_1694032401399_ns2_test-16940322964961 2023-09-06T20:33:28,234 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1, snapshotDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032401399_ns2_test-16940322964961. 2023-09-06T20:33:28,240 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032401399_ns2_test-16940322964961 to hdfs://localhost:39327/backupUT/backup_1694032399062/ns2/test-16940322964961/.hbase-snapshot/.tmp/snapshot_1694032401399_ns2_test-16940322964961 2023-09-06T20:33:28,652 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:28,653 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:28,653 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:28,653 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:29,661 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop-347991400830158678.jar 2023-09-06T20:33:29,661 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:29,662 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:29,662 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:29,663 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:29,663 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:29,663 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:29,664 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:29,664 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:29,664 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:29,665 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:33:29,666 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:33:29,666 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:33:29,666 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:33:29,667 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:33:29,667 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:33:29,667 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:33:29,668 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:33:29,668 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:33:29,668 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:33:29,669 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:33:29,669 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:33:29,670 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:33:29,670 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:33:29,671 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:33:29,671 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:33:29,671 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:33:29,672 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:33:29,672 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:33:29,714 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1694032401399_ns2_test-16940322964961' hfile list 2023-09-06T20:33:29,715 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-09-06T20:33:29,786 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032409750/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:33:29,786 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032409750/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:33:29,841 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-09-06T20:33:29,866 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc/f/5f020f30827940489104cc2b9277b6d8_SeqId_4_ output=hdfs://localhost:39327/backupUT/backup_1694032399062/ns2/test-16940322964961/archive/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc/f/5f020f30827940489104cc2b9277b6d8_SeqId_4_ 2023-09-06T20:33:29,867 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 8.320M/sec 2023-09-06T20:33:29,883 DEBUG [regionserver/jenkins-hbase3:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.38461538461538464, tune throughput to 69.23 MB/second 2023-09-06T20:33:30,818 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-09-06T20:33:30,819 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-09-06T20:33:30,825 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1694032401399_ns2_test-16940322964961 2023-09-06T20:33:30,825 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1694032401399_ns2_test-16940322964961 finished. 2023-09-06T20:33:30,825 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1014): ns3:test-16940322964962 2023-09-06T20:33:30,825 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1014): ns2:test-16940322964961 2023-09-06T20:33:30,831 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1694032399062 2023-09-06T20:33:30,832 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-09-06T20:33:31,237 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:39327/backupUT/backup_1694032399062/.backup.manifest 2023-09-06T20:33:31,237 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1694032399062 2023-09-06T20:33:31,237 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-09-06T20:33:31,642 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:39327/backupUT/backup_1694032399062/.backup.manifest 2023-09-06T20:33:31,642 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(358): Backup backup_1694032399062 finished: type=FULL,tablelist=ns3:test-16940322964962;ns2:test-16940322964961,targetRootDir=hdfs://localhost:39327/backupUT,startts=1694032401208,completets=1694032410831,bytescopied=0 2023-09-06T20:33:31,642 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-09-06T20:33:31,642 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1694032403518_ns3_test-16940322964962 2023-09-06T20:33:31,643 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1694032403518_ns3_test-16940322964962" 2023-09-06T20:33:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1694032403518_ns3_test-16940322964962 2023-09-06T20:33:31,646 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1694032403518_ns3_test-16940322964962 for backup backup_1694032399062 succeeded. 2023-09-06T20:33:31,646 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1694032401399_ns2_test-16940322964961 2023-09-06T20:33:31,646 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1694032401399_ns2_test-16940322964961" 2023-09-06T20:33:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1694032401399_ns2_test-16940322964961 2023-09-06T20:33:31,649 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1694032401399_ns2_test-16940322964961 for backup backup_1694032399062 succeeded. 2023-09-06T20:33:31,649 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:33:31,653 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:33:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:33:31,655 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:33:31,656 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:33:31,662 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(376): Backup backup_1694032399062 completed. 2023-09-06T20:33:31,662 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:33:31,672 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(101): HBase table ns2:table2 does not exist. It will be created during restore process 2023-09-06T20:33:31,673 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(101): HBase table ns3:table3_restore does not exist. It will be created during restore process 2023-09-06T20:33:31,673 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:39327/backupUT/backup_1694032399062/.backup.manifest 2023-09-06T20:33:31,675 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032399062/.backup.manifest 2023-09-06T20:33:31,676 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:39327/backupUT/backup_1694032399062/.backup.manifest 2023-09-06T20:33:31,677 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032399062/.backup.manifest 2023-09-06T20:33:31,678 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:39327/backupUT/backup_1694032399062/.backup.manifest 2023-09-06T20:33:31,679 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032399062/.backup.manifest 2023-09-06T20:33:31,679 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(148): Restoring 'ns2:test-16940322964961' to 'ns2:table2' from full backup image hdfs://localhost:39327/backupUT/backup_1694032399062/ns2/test-16940322964961 2023-09-06T20:33:31,685 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(487): Creating target table 'ns2:table2' 2023-09-06T20:33:31,686 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:39327/backupUT/backup_1694032399062/ns2/test-16940322964961/archive/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc 2023-09-06T20:33:31,686 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:39327/backupUT/backup_1694032399062/ns2/test-16940322964961/archive/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc/f in region [hdfs://localhost:39327/backupUT/backup_1694032399062/ns2/test-16940322964961/archive/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc] 2023-09-06T20:33:31,689 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:39327/backupUT/backup_1694032399062/ns2/test-16940322964961/archive/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc/f/5f020f30827940489104cc2b9277b6d8_SeqId_4_ first=row0 last=row98 2023-09-06T20:33:31,690 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:33:31,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=101, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:table2 2023-09-06T20:33:31,692 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:33:31,692 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "table2" procId is: 101 2023-09-06T20:33:31,693 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:33:31,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=101 2023-09-06T20:33:31,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=101 2023-09-06T20:33:31,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=101 2023-09-06T20:33:32,099 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 3b428ffdfb7a12a6dafc1434d4a6b7ce, NAME => 'ns2:table2,,1694032411690.3b428ffdfb7a12a6dafc1434d4a6b7ce.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:33:32,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=101 2023-09-06T20:33:32,505 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:table2,,1694032411690.3b428ffdfb7a12a6dafc1434d4a6b7ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:33:32,505 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1668): Closing 3b428ffdfb7a12a6dafc1434d4a6b7ce, disabling compactions & flushes 2023-09-06T20:33:32,505 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:table2,,1694032411690.3b428ffdfb7a12a6dafc1434d4a6b7ce. 2023-09-06T20:33:32,505 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:table2,,1694032411690.3b428ffdfb7a12a6dafc1434d4a6b7ce. 2023-09-06T20:33:32,505 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:table2,,1694032411690.3b428ffdfb7a12a6dafc1434d4a6b7ce. after waiting 0 ms 2023-09-06T20:33:32,505 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:table2,,1694032411690.3b428ffdfb7a12a6dafc1434d4a6b7ce. 2023-09-06T20:33:32,505 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1909): Closed ns2:table2,,1694032411690.3b428ffdfb7a12a6dafc1434d4a6b7ce. 2023-09-06T20:33:32,505 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1622): Region close journal for 3b428ffdfb7a12a6dafc1434d4a6b7ce: 2023-09-06T20:33:32,506 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:33:32,506 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:table2,,1694032411690.3b428ffdfb7a12a6dafc1434d4a6b7ce.","families":{"info":[{"qualifier":"regioninfo","vlen":36,"tag":[],"timestamp":"1694032412506"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032412506"}]},"ts":"1694032412506"} 2023-09-06T20:33:32,508 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:33:32,508 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:33:32,508 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032412508"}]},"ts":"1694032412508"} 2023-09-06T20:33:32,509 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=ENABLING in hbase:meta 2023-09-06T20:33:32,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:table2, region=3b428ffdfb7a12a6dafc1434d4a6b7ce, ASSIGN}] 2023-09-06T20:33:32,515 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:table2, region=3b428ffdfb7a12a6dafc1434d4a6b7ce, ASSIGN 2023-09-06T20:33:32,515 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:table2, region=3b428ffdfb7a12a6dafc1434d4a6b7ce, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36961,1694032287907; forceNewPlan=false, retain=false 2023-09-06T20:33:32,667 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=102 updating hbase:meta row=3b428ffdfb7a12a6dafc1434d4a6b7ce, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:32,669 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE; OpenRegionProcedure 3b428ffdfb7a12a6dafc1434d4a6b7ce, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:33:32,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=101 2023-09-06T20:33:32,825 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] handler.AssignRegionHandler(130): Open ns2:table2,,1694032411690.3b428ffdfb7a12a6dafc1434d4a6b7ce. 2023-09-06T20:33:32,825 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(7254): Opening region: {ENCODED => 3b428ffdfb7a12a6dafc1434d4a6b7ce, NAME => 'ns2:table2,,1694032411690.3b428ffdfb7a12a6dafc1434d4a6b7ce.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:33:32,826 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:33:32,826 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table2 3b428ffdfb7a12a6dafc1434d4a6b7ce 2023-09-06T20:33:32,826 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(888): Instantiated ns2:table2,,1694032411690.3b428ffdfb7a12a6dafc1434d4a6b7ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:33:32,826 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(7296): checking encryption for 3b428ffdfb7a12a6dafc1434d4a6b7ce 2023-09-06T20:33:32,826 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(7299): checking classloading for 3b428ffdfb7a12a6dafc1434d4a6b7ce 2023-09-06T20:33:32,827 INFO [StoreOpener-3b428ffdfb7a12a6dafc1434d4a6b7ce-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 3b428ffdfb7a12a6dafc1434d4a6b7ce 2023-09-06T20:33:32,828 INFO [StoreOpener-3b428ffdfb7a12a6dafc1434d4a6b7ce-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3b428ffdfb7a12a6dafc1434d4a6b7ce columnFamilyName f 2023-09-06T20:33:32,829 DEBUG [StoreOpener-3b428ffdfb7a12a6dafc1434d4a6b7ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:33:32,829 INFO [StoreOpener-3b428ffdfb7a12a6dafc1434d4a6b7ce-1 {}] regionserver.HStore(324): Store=3b428ffdfb7a12a6dafc1434d4a6b7ce/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:33:32,830 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/table2/3b428ffdfb7a12a6dafc1434d4a6b7ce 2023-09-06T20:33:32,830 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/table2/3b428ffdfb7a12a6dafc1434d4a6b7ce 2023-09-06T20:33:32,833 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1076): writing seq id for 3b428ffdfb7a12a6dafc1434d4a6b7ce 2023-09-06T20:33:32,836 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/table2/3b428ffdfb7a12a6dafc1434d4a6b7ce/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:33:32,836 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1093): Opened 3b428ffdfb7a12a6dafc1434d4a6b7ce; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10282730560, jitterRate=-0.04234609007835388}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:33:32,837 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(992): Region open journal for 3b428ffdfb7a12a6dafc1434d4a6b7ce: 2023-09-06T20:33:32,838 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:table2,,1694032411690.3b428ffdfb7a12a6dafc1434d4a6b7ce., pid=103, masterSystemTime=1694032412821 2023-09-06T20:33:32,840 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:table2,,1694032411690.3b428ffdfb7a12a6dafc1434d4a6b7ce. 2023-09-06T20:33:32,840 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] handler.AssignRegionHandler(158): Opened ns2:table2,,1694032411690.3b428ffdfb7a12a6dafc1434d4a6b7ce. 2023-09-06T20:33:32,840 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=102 updating hbase:meta row=3b428ffdfb7a12a6dafc1434d4a6b7ce, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:32,843 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=103, resume processing ppid=102 2023-09-06T20:33:32,843 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=103, ppid=102, state=SUCCESS; OpenRegionProcedure 3b428ffdfb7a12a6dafc1434d4a6b7ce, server=jenkins-hbase3.apache.org,36961,1694032287907 in 173 msec 2023-09-06T20:33:32,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=102, resume processing ppid=101 2023-09-06T20:33:32,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=102, ppid=101, state=SUCCESS; TransitRegionStateProcedure table=ns2:table2, region=3b428ffdfb7a12a6dafc1434d4a6b7ce, ASSIGN in 330 msec 2023-09-06T20:33:32,846 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:33:32,846 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032412846"}]},"ts":"1694032412846"} 2023-09-06T20:33:32,847 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=ENABLED in hbase:meta 2023-09-06T20:33:32,850 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:33:32,852 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=101, state=SUCCESS; CreateTableProcedure table=ns2:table2 in 1.1600 sec 2023-09-06T20:33:33,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=101 2023-09-06T20:33:33,801 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:39327/backupUT/backup_1694032399062/ns2/test-16940322964961/archive/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc from hbase tables ns2:test-16940322964961 to tables ns2:table2 2023-09-06T20:33:33,801 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-16940322964961 into ns2:table2 2023-09-06T20:33:33,803 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-table2-1694032413801 from hdfs://localhost:39327/backupUT/backup_1694032399062/ns2/test-16940322964961/archive/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc 2023-09-06T20:33:33,809 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@261bac1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:33,809 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:33:33,810 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:53172, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:33:33,811 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-09-06T20:33:33,811 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:table2 2023-09-06T20:33:33,814 DEBUG [hconnection-0x621cce7a-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:33,817 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48936, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:33:33,818 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:table2: 2023-09-06T20:33:33,818 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-09-06T20:33:33,818 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/partitions_269f0a98-a5df-41be-9792-e062e6b31720 2023-09-06T20:33:34,225 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:34,226 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:34,226 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:34,227 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:35,191 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop-6783191858676775012.jar 2023-09-06T20:33:35,192 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:35,192 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:35,192 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:35,193 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:35,193 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:35,194 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:35,194 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:35,194 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:35,195 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:35,195 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:33:35,196 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:33:35,196 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:33:35,197 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:33:35,197 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:33:35,198 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:33:35,198 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:33:35,198 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:33:35,199 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:33:35,199 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:33:35,199 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:33:35,200 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:33:35,200 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:35,201 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:35,201 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:35,201 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:35,201 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:35,202 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:35,202 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:33:35,203 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:table2 2023-09-06T20:33:35,203 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:33:35,203 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-09-06T20:33:35,203 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-09-06T20:33:35,204 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:33:35,216 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-09-06T20:33:35,296 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032415261/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:33:35,296 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032415261/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:33:35,365 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:39327/backupUT/backup_1694032399062/ns2/test-16940322964961/archive/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc/f/5f020f30827940489104cc2b9277b6d8_SeqId_4_ 2023-09-06T20:33:35,368 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-09-06T20:33:35,387 DEBUG [pool-396-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34faacf7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:35,388 DEBUG [hconnection-0x5e7568b9-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:35,390 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48940, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:33:35,391 DEBUG [pool-396-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:33:35,392 DEBUG [pool-396-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-09-06T20:33:35,392 DEBUG [pool-396-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-09-06T20:33:35,397 DEBUG [pool-396-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-table2-1694032413801/_temporary/0/_temporary/attempt_local1399617244_0016_r_000000_0/f/1dbd2e32bf75429db2eb97f55da07149 is 37, key is row10/f:q1/1694032300351/Put/seqid=0 2023-09-06T20:33:35,398 WARN [Thread-1715 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:36961] Specified favored nodes: [jenkins-hbase3.apache.org:36961] 2023-09-06T20:33:35,747 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_table2 2023-09-06T20:33:35,747 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_table2 Metrics about Tables on a single HBase RegionServer 2023-09-06T20:33:36,250 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:table2' 2023-09-06T20:33:36,366 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-table2-1694032413801 2023-09-06T20:33:36,371 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78422811, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:36,372 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:36,374 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48950, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:33:36,379 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-table2-1694032413801/_SUCCESS 2023-09-06T20:33:36,382 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:33:36,384 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:53174, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:33:36,392 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-table2-1694032413801/f/1dbd2e32bf75429db2eb97f55da07149 first=Optional[row0] last=Optional[row98] 2023-09-06T20:33:36,393 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=ns2:table2,,1694032411690.3b428ffdfb7a12a6dafc1434d4a6b7ce., hostname=jenkins-hbase3.apache.org,36961,1694032287907, seqNum=2 for row with hfile group [{f,hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-table2-1694032413801/f/1dbd2e32bf75429db2eb97f55da07149}] 2023-09-06T20:33:36,398 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-table2-1694032413801/f/1dbd2e32bf75429db2eb97f55da07149 for inclusion in 3b428ffdfb7a12a6dafc1434d4a6b7ce/f 2023-09-06T20:33:36,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-09-06T20:33:36,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(625): Region bounds: first= last= 2023-09-06T20:33:36,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HRegion(2520): Flush status journal for 3b428ffdfb7a12a6dafc1434d4a6b7ce: 2023-09-06T20:33:36,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-table2-1694032413801/f/1dbd2e32bf75429db2eb97f55da07149 to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__ns2_table2__fd246jpkrqdtnoqk877v2fvqc15816bev0npvmild69vehl1v20hcgr3h4ni9lnc/f/1dbd2e32bf75429db2eb97f55da07149 2023-09-06T20:33:36,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__ns2_table2__fd246jpkrqdtnoqk877v2fvqc15816bev0npvmild69vehl1v20hcgr3h4ni9lnc/f/1dbd2e32bf75429db2eb97f55da07149 as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/table2/3b428ffdfb7a12a6dafc1434d4a6b7ce/f/80d87a2c1c5041c7b80d82e124fb0476_SeqId_4_ 2023-09-06T20:33:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4110347a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:33:36,412 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:53180, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:33:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:36,416 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48956, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:33:36,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:33:36,431 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__ns2_table2__fd246jpkrqdtnoqk877v2fvqc15816bev0npvmild69vehl1v20hcgr3h4ni9lnc/f/1dbd2e32bf75429db2eb97f55da07149 into 3b428ffdfb7a12a6dafc1434d4a6b7ce/f as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/table2/3b428ffdfb7a12a6dafc1434d4a6b7ce/f/80d87a2c1c5041c7b80d82e124fb0476_SeqId_4_ - updating store file list. 2023-09-06T20:33:36,436 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/table2/3b428ffdfb7a12a6dafc1434d4a6b7ce/f/80d87a2c1c5041c7b80d82e124fb0476_SeqId_4_ into 3b428ffdfb7a12a6dafc1434d4a6b7ce/f 2023-09-06T20:33:36,436 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__ns2_table2__fd246jpkrqdtnoqk877v2fvqc15816bev0npvmild69vehl1v20hcgr3h4ni9lnc/f/1dbd2e32bf75429db2eb97f55da07149 into 3b428ffdfb7a12a6dafc1434d4a6b7ce/f (new location: hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/table2/3b428ffdfb7a12a6dafc1434d4a6b7ce/f/80d87a2c1c5041c7b80d82e124fb0476_SeqId_4_) 2023-09-06T20:33:36,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__ns2_table2__fd246jpkrqdtnoqk877v2fvqc15816bev0npvmild69vehl1v20hcgr3h4ni9lnc/f/1dbd2e32bf75429db2eb97f55da07149 2023-09-06T20:33:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b15ff64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:33:36,446 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:53192, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-09-06T20:33:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:36,450 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48964, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-09-06T20:33:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:33:36,465 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:33:36,465 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-09-06T20:33:36,465 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-09-06T20:33:36,465 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(233): Backup: backup_1694032399062 hdfs://localhost:39327/backupUT/backup_1694032399062/ns2/test-16940322964961/ 2023-09-06T20:33:36,466 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:39327/backupUT/backup_1694032399062/.backup.manifest 2023-09-06T20:33:36,469 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032399062/.backup.manifest 2023-09-06T20:33:36,469 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(148): Restoring 'ns3:test-16940322964962' to 'ns3:table3_restore' from full backup image hdfs://localhost:39327/backupUT/backup_1694032399062/ns3/test-16940322964962 2023-09-06T20:33:36,475 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(101): Folder tableArchivePath: hdfs://localhost:39327/backupUT/backup_1694032399062/ns3/test-16940322964962/archive/data/ns3/test-16940322964962 does not exists 2023-09-06T20:33:36,475 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(326): find table descriptor but no archive dir for table ns3:test-16940322964962, will only create table 2023-09-06T20:33:36,476 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(487): Creating target table 'ns3:table3_restore' 2023-09-06T20:33:36,477 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns3:table3_restore', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:33:36,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=104, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:table3_restore 2023-09-06T20:33:36,479 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:33:36,480 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "table3_restore" procId is: 104 2023-09-06T20:33:36,480 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:33:36,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=104 2023-09-06T20:33:36,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=104 2023-09-06T20:33:36,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=104 2023-09-06T20:33:36,888 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 8f0b416e550a6ed13b83e617e36ebcae, NAME => 'ns3:table3_restore,,1694032416476.8f0b416e550a6ed13b83e617e36ebcae.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:table3_restore', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:33:37,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=104 2023-09-06T20:33:37,295 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:table3_restore,,1694032416476.8f0b416e550a6ed13b83e617e36ebcae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:33:37,295 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1668): Closing 8f0b416e550a6ed13b83e617e36ebcae, disabling compactions & flushes 2023-09-06T20:33:37,295 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:table3_restore,,1694032416476.8f0b416e550a6ed13b83e617e36ebcae. 2023-09-06T20:33:37,295 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:table3_restore,,1694032416476.8f0b416e550a6ed13b83e617e36ebcae. 2023-09-06T20:33:37,295 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:table3_restore,,1694032416476.8f0b416e550a6ed13b83e617e36ebcae. after waiting 0 ms 2023-09-06T20:33:37,295 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:table3_restore,,1694032416476.8f0b416e550a6ed13b83e617e36ebcae. 2023-09-06T20:33:37,295 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1909): Closed ns3:table3_restore,,1694032416476.8f0b416e550a6ed13b83e617e36ebcae. 2023-09-06T20:33:37,295 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1622): Region close journal for 8f0b416e550a6ed13b83e617e36ebcae: 2023-09-06T20:33:37,296 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:33:37,296 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:table3_restore,,1694032416476.8f0b416e550a6ed13b83e617e36ebcae.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1694032417296"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032417296"}]},"ts":"1694032417296"} 2023-09-06T20:33:37,298 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:33:37,298 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:33:37,298 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032417298"}]},"ts":"1694032417298"} 2023-09-06T20:33:37,299 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=ENABLING in hbase:meta 2023-09-06T20:33:37,307 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:table3_restore, region=8f0b416e550a6ed13b83e617e36ebcae, ASSIGN}] 2023-09-06T20:33:37,309 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:table3_restore, region=8f0b416e550a6ed13b83e617e36ebcae, ASSIGN 2023-09-06T20:33:37,309 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:table3_restore, region=8f0b416e550a6ed13b83e617e36ebcae, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36961,1694032287907; forceNewPlan=false, retain=false 2023-09-06T20:33:37,461 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=105 updating hbase:meta row=8f0b416e550a6ed13b83e617e36ebcae, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:37,462 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE; OpenRegionProcedure 8f0b416e550a6ed13b83e617e36ebcae, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:33:37,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=104 2023-09-06T20:33:37,617 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(130): Open ns3:table3_restore,,1694032416476.8f0b416e550a6ed13b83e617e36ebcae. 2023-09-06T20:33:37,617 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7254): Opening region: {ENCODED => 8f0b416e550a6ed13b83e617e36ebcae, NAME => 'ns3:table3_restore,,1694032416476.8f0b416e550a6ed13b83e617e36ebcae.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:33:37,618 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:33:37,618 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table3_restore 8f0b416e550a6ed13b83e617e36ebcae 2023-09-06T20:33:37,618 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(888): Instantiated ns3:table3_restore,,1694032416476.8f0b416e550a6ed13b83e617e36ebcae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:33:37,618 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7296): checking encryption for 8f0b416e550a6ed13b83e617e36ebcae 2023-09-06T20:33:37,618 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7299): checking classloading for 8f0b416e550a6ed13b83e617e36ebcae 2023-09-06T20:33:37,619 INFO [StoreOpener-8f0b416e550a6ed13b83e617e36ebcae-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 8f0b416e550a6ed13b83e617e36ebcae 2023-09-06T20:33:37,620 INFO [StoreOpener-8f0b416e550a6ed13b83e617e36ebcae-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f0b416e550a6ed13b83e617e36ebcae columnFamilyName f 2023-09-06T20:33:37,620 DEBUG [StoreOpener-8f0b416e550a6ed13b83e617e36ebcae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:33:37,621 INFO [StoreOpener-8f0b416e550a6ed13b83e617e36ebcae-1 {}] regionserver.HStore(324): Store=8f0b416e550a6ed13b83e617e36ebcae/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:33:37,621 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/table3_restore/8f0b416e550a6ed13b83e617e36ebcae 2023-09-06T20:33:37,622 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/table3_restore/8f0b416e550a6ed13b83e617e36ebcae 2023-09-06T20:33:37,624 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1076): writing seq id for 8f0b416e550a6ed13b83e617e36ebcae 2023-09-06T20:33:37,625 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/table3_restore/8f0b416e550a6ed13b83e617e36ebcae/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:33:37,626 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1093): Opened 8f0b416e550a6ed13b83e617e36ebcae; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10626205440, jitterRate=-0.010357499122619629}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:33:37,626 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(992): Region open journal for 8f0b416e550a6ed13b83e617e36ebcae: 2023-09-06T20:33:37,627 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:table3_restore,,1694032416476.8f0b416e550a6ed13b83e617e36ebcae., pid=106, masterSystemTime=1694032417614 2023-09-06T20:33:37,628 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:table3_restore,,1694032416476.8f0b416e550a6ed13b83e617e36ebcae. 2023-09-06T20:33:37,628 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(158): Opened ns3:table3_restore,,1694032416476.8f0b416e550a6ed13b83e617e36ebcae. 2023-09-06T20:33:37,629 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=105 updating hbase:meta row=8f0b416e550a6ed13b83e617e36ebcae, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:37,631 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=106, resume processing ppid=105 2023-09-06T20:33:37,631 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=106, ppid=105, state=SUCCESS; OpenRegionProcedure 8f0b416e550a6ed13b83e617e36ebcae, server=jenkins-hbase3.apache.org,36961,1694032287907 in 168 msec 2023-09-06T20:33:37,632 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=105, resume processing ppid=104 2023-09-06T20:33:37,632 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=105, ppid=104, state=SUCCESS; TransitRegionStateProcedure table=ns3:table3_restore, region=8f0b416e550a6ed13b83e617e36ebcae, ASSIGN in 327 msec 2023-09-06T20:33:37,632 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:33:37,632 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032417632"}]},"ts":"1694032417632"} 2023-09-06T20:33:37,633 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=ENABLED in hbase:meta 2023-09-06T20:33:37,635 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:33:37,636 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=104, state=SUCCESS; CreateTableProcedure table=ns3:table3_restore in 1.1580 sec 2023-09-06T20:33:38,378 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:33:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=104 2023-09-06T20:33:38,589 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-09-06T20:33:38,589 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(233): Backup: backup_1694032399062 hdfs://localhost:39327/backupUT/backup_1694032399062/ns3/test-16940322964962/ 2023-09-06T20:33:38,590 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-09-06T20:33:38,591 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable ns2:table2 2023-09-06T20:33:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns2:table2 2023-09-06T20:33:38,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=107 2023-09-06T20:33:38,595 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032418595"}]},"ts":"1694032418595"} 2023-09-06T20:33:38,596 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=DISABLING in hbase:meta 2023-09-06T20:33:38,598 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(293): Set ns2:table2 to state=DISABLING 2023-09-06T20:33:38,598 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:table2, region=3b428ffdfb7a12a6dafc1434d4a6b7ce, UNASSIGN}] 2023-09-06T20:33:38,600 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=108, ppid=107, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:table2, region=3b428ffdfb7a12a6dafc1434d4a6b7ce, UNASSIGN 2023-09-06T20:33:38,601 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=108 updating hbase:meta row=3b428ffdfb7a12a6dafc1434d4a6b7ce, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:38,602 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE; CloseRegionProcedure 3b428ffdfb7a12a6dafc1434d4a6b7ce, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:33:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=107 2023-09-06T20:33:38,754 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] handler.UnassignRegionHandler(119): Close 3b428ffdfb7a12a6dafc1434d4a6b7ce 2023-09-06T20:33:38,754 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1668): Closing 3b428ffdfb7a12a6dafc1434d4a6b7ce, disabling compactions & flushes 2023-09-06T20:33:38,754 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1690): Closing region ns2:table2,,1694032411690.3b428ffdfb7a12a6dafc1434d4a6b7ce. 2023-09-06T20:33:38,754 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:table2,,1694032411690.3b428ffdfb7a12a6dafc1434d4a6b7ce. 2023-09-06T20:33:38,754 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1778): Acquired close lock on ns2:table2,,1694032411690.3b428ffdfb7a12a6dafc1434d4a6b7ce. after waiting 0 ms 2023-09-06T20:33:38,754 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1788): Updates disabled for region ns2:table2,,1694032411690.3b428ffdfb7a12a6dafc1434d4a6b7ce. 2023-09-06T20:33:38,760 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/table2/3b428ffdfb7a12a6dafc1434d4a6b7ce/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-09-06T20:33:38,761 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:33:38,762 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1909): Closed ns2:table2,,1694032411690.3b428ffdfb7a12a6dafc1434d4a6b7ce. 2023-09-06T20:33:38,762 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1622): Region close journal for 3b428ffdfb7a12a6dafc1434d4a6b7ce: 2023-09-06T20:33:38,763 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] handler.UnassignRegionHandler(163): Closed 3b428ffdfb7a12a6dafc1434d4a6b7ce 2023-09-06T20:33:38,763 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=108 updating hbase:meta row=3b428ffdfb7a12a6dafc1434d4a6b7ce, regionState=CLOSED 2023-09-06T20:33:38,766 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=109, resume processing ppid=108 2023-09-06T20:33:38,766 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=109, ppid=108, state=SUCCESS; CloseRegionProcedure 3b428ffdfb7a12a6dafc1434d4a6b7ce, server=jenkins-hbase3.apache.org,36961,1694032287907 in 163 msec 2023-09-06T20:33:38,767 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=108, resume processing ppid=107 2023-09-06T20:33:38,767 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=108, ppid=107, state=SUCCESS; TransitRegionStateProcedure table=ns2:table2, region=3b428ffdfb7a12a6dafc1434d4a6b7ce, UNASSIGN in 168 msec 2023-09-06T20:33:38,768 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032418768"}]},"ts":"1694032418768"} 2023-09-06T20:33:38,769 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=DISABLED in hbase:meta 2023-09-06T20:33:38,770 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set ns2:table2 to state=DISABLED 2023-09-06T20:33:38,772 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=107, state=SUCCESS; DisableTableProcedure table=ns2:table2 in 179 msec 2023-09-06T20:33:38,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=107 2023-09-06T20:33:38,897 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$5(2448): Client=jenkins//172.31.12.81 delete ns2:table2 2023-09-06T20:33:38,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=110, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=ns2:table2 2023-09-06T20:33:38,900 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=110, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=ns2:table2 2023-09-06T20:33:38,901 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=110, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=ns2:table2 2023-09-06T20:33:38,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=110 2023-09-06T20:33:38,903 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/table2/3b428ffdfb7a12a6dafc1434d4a6b7ce 2023-09-06T20:33:38,904 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/table2/3b428ffdfb7a12a6dafc1434d4a6b7ce/f, FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/table2/3b428ffdfb7a12a6dafc1434d4a6b7ce/recovered.edits] 2023-09-06T20:33:38,909 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/table2/3b428ffdfb7a12a6dafc1434d4a6b7ce/f/80d87a2c1c5041c7b80d82e124fb0476_SeqId_4_ to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/archive/data/ns2/table2/3b428ffdfb7a12a6dafc1434d4a6b7ce/f/80d87a2c1c5041c7b80d82e124fb0476_SeqId_4_ 2023-09-06T20:33:38,912 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/table2/3b428ffdfb7a12a6dafc1434d4a6b7ce/recovered.edits/6.seqid to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/archive/data/ns2/table2/3b428ffdfb7a12a6dafc1434d4a6b7ce/recovered.edits/6.seqid 2023-09-06T20:33:38,912 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/table2/3b428ffdfb7a12a6dafc1434d4a6b7ce 2023-09-06T20:33:38,913 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(293): Archived ns2:table2 regions 2023-09-06T20:33:38,915 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=110, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=ns2:table2 2023-09-06T20:33:38,917 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns2:table2 from hbase:meta 2023-09-06T20:33:38,919 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(387): Removing 'ns2:table2' descriptor. 2023-09-06T20:33:38,920 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=110, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=ns2:table2 2023-09-06T20:33:38,920 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(377): Removing 'ns2:table2' from region states. 2023-09-06T20:33:38,920 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:table2,,1694032411690.3b428ffdfb7a12a6dafc1434d4a6b7ce.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1694032418920"}]},"ts":"9223372036854775807"} 2023-09-06T20:33:38,925 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-06T20:33:38,925 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 3b428ffdfb7a12a6dafc1434d4a6b7ce, NAME => 'ns2:table2,,1694032411690.3b428ffdfb7a12a6dafc1434d4a6b7ce.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:33:38,925 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(381): Marking 'ns2:table2' as deleted. 2023-09-06T20:33:38,925 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1694032418925"}]},"ts":"9223372036854775807"} 2023-09-06T20:33:38,927 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table ns2:table2 state from META 2023-09-06T20:33:38,930 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(127): Finished pid=110, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=ns2:table2 2023-09-06T20:33:38,931 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=110, state=SUCCESS; DeleteTableProcedure table=ns2:table2 in 32 msec 2023-09-06T20:33:39,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=110 2023-09-06T20:33:39,003 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable ns3:table3_restore 2023-09-06T20:33:39,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns3:table3_restore 2023-09-06T20:33:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=111 2023-09-06T20:33:39,007 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032419007"}]},"ts":"1694032419007"} 2023-09-06T20:33:39,008 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=DISABLING in hbase:meta 2023-09-06T20:33:39,010 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(293): Set ns3:table3_restore to state=DISABLING 2023-09-06T20:33:39,010 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:table3_restore, region=8f0b416e550a6ed13b83e617e36ebcae, UNASSIGN}] 2023-09-06T20:33:39,012 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=112, ppid=111, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:table3_restore, region=8f0b416e550a6ed13b83e617e36ebcae, UNASSIGN 2023-09-06T20:33:39,013 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=112 updating hbase:meta row=8f0b416e550a6ed13b83e617e36ebcae, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:39,014 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE; CloseRegionProcedure 8f0b416e550a6ed13b83e617e36ebcae, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:33:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=111 2023-09-06T20:33:39,166 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] handler.UnassignRegionHandler(119): Close 8f0b416e550a6ed13b83e617e36ebcae 2023-09-06T20:33:39,167 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1668): Closing 8f0b416e550a6ed13b83e617e36ebcae, disabling compactions & flushes 2023-09-06T20:33:39,167 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1690): Closing region ns3:table3_restore,,1694032416476.8f0b416e550a6ed13b83e617e36ebcae. 2023-09-06T20:33:39,167 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:table3_restore,,1694032416476.8f0b416e550a6ed13b83e617e36ebcae. 2023-09-06T20:33:39,167 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1778): Acquired close lock on ns3:table3_restore,,1694032416476.8f0b416e550a6ed13b83e617e36ebcae. after waiting 0 ms 2023-09-06T20:33:39,167 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1788): Updates disabled for region ns3:table3_restore,,1694032416476.8f0b416e550a6ed13b83e617e36ebcae. 2023-09-06T20:33:39,171 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/table3_restore/8f0b416e550a6ed13b83e617e36ebcae/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-09-06T20:33:39,171 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:33:39,172 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1909): Closed ns3:table3_restore,,1694032416476.8f0b416e550a6ed13b83e617e36ebcae. 2023-09-06T20:33:39,172 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1622): Region close journal for 8f0b416e550a6ed13b83e617e36ebcae: 2023-09-06T20:33:39,174 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] handler.UnassignRegionHandler(163): Closed 8f0b416e550a6ed13b83e617e36ebcae 2023-09-06T20:33:39,174 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=112 updating hbase:meta row=8f0b416e550a6ed13b83e617e36ebcae, regionState=CLOSED 2023-09-06T20:33:39,176 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=113, resume processing ppid=112 2023-09-06T20:33:39,176 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=113, ppid=112, state=SUCCESS; CloseRegionProcedure 8f0b416e550a6ed13b83e617e36ebcae, server=jenkins-hbase3.apache.org,36961,1694032287907 in 161 msec 2023-09-06T20:33:39,177 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=112, resume processing ppid=111 2023-09-06T20:33:39,177 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=112, ppid=111, state=SUCCESS; TransitRegionStateProcedure table=ns3:table3_restore, region=8f0b416e550a6ed13b83e617e36ebcae, UNASSIGN in 166 msec 2023-09-06T20:33:39,178 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032419178"}]},"ts":"1694032419178"} 2023-09-06T20:33:39,179 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=DISABLED in hbase:meta 2023-09-06T20:33:39,180 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(305): Set ns3:table3_restore to state=DISABLED 2023-09-06T20:33:39,182 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=111, state=SUCCESS; DisableTableProcedure table=ns3:table3_restore in 178 msec 2023-09-06T20:33:39,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=111 2023-09-06T20:33:39,310 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$5(2448): Client=jenkins//172.31.12.81 delete ns3:table3_restore 2023-09-06T20:33:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=114, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=ns3:table3_restore 2023-09-06T20:33:39,312 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=114, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-09-06T20:33:39,313 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=114, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-09-06T20:33:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=114 2023-09-06T20:33:39,315 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/table3_restore/8f0b416e550a6ed13b83e617e36ebcae 2023-09-06T20:33:39,316 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/table3_restore/8f0b416e550a6ed13b83e617e36ebcae/f, FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/table3_restore/8f0b416e550a6ed13b83e617e36ebcae/recovered.edits] 2023-09-06T20:33:39,321 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/table3_restore/8f0b416e550a6ed13b83e617e36ebcae/recovered.edits/4.seqid to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/archive/data/ns3/table3_restore/8f0b416e550a6ed13b83e617e36ebcae/recovered.edits/4.seqid 2023-09-06T20:33:39,322 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/table3_restore/8f0b416e550a6ed13b83e617e36ebcae 2023-09-06T20:33:39,322 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(293): Archived ns3:table3_restore regions 2023-09-06T20:33:39,324 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=114, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-09-06T20:33:39,326 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns3:table3_restore from hbase:meta 2023-09-06T20:33:39,327 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(387): Removing 'ns3:table3_restore' descriptor. 2023-09-06T20:33:39,328 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=114, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-09-06T20:33:39,328 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(377): Removing 'ns3:table3_restore' from region states. 2023-09-06T20:33:39,329 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:table3_restore,,1694032416476.8f0b416e550a6ed13b83e617e36ebcae.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1694032419328"}]},"ts":"9223372036854775807"} 2023-09-06T20:33:39,330 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-06T20:33:39,330 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 8f0b416e550a6ed13b83e617e36ebcae, NAME => 'ns3:table3_restore,,1694032416476.8f0b416e550a6ed13b83e617e36ebcae.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:33:39,330 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(381): Marking 'ns3:table3_restore' as deleted. 2023-09-06T20:33:39,330 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1694032419330"}]},"ts":"9223372036854775807"} 2023-09-06T20:33:39,332 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1678): Deleted table ns3:table3_restore state from META 2023-09-06T20:33:39,334 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(127): Finished pid=114, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-09-06T20:33:39,334 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=114, state=SUCCESS; DeleteTableProcedure table=ns3:table3_restore in 24 msec 2023-09-06T20:33:39,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=114 2023-09-06T20:33:39,432 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultiple Thread=601 (was 597) - Thread LEAK? -, OpenFileDescriptor=675 (was 682), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=118 (was 143), ProcessCount=167 (was 167), AvailableMemoryMB=3042 (was 3102) 2023-09-06T20:33:39,432 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(130): Thread=601 is superior to 500 2023-09-06T20:33:39,446 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultipleOverwriteCommand Thread=601, OpenFileDescriptor=675, MaxFileDescriptor=60000, SystemLoadAverage=118, ProcessCount=167, AvailableMemoryMB=3042 2023-09-06T20:33:39,446 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(130): Thread=601 is superior to 500 2023-09-06T20:33:39,446 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] backup.TestFullRestore(234): create full backup image on multiple tables: command-line 2023-09-06T20:33:39,452 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123a5191, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:39,455 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:39,457 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48976, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:33:39,459 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:33:39,460 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:53196, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:33:39,469 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:33:39,474 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:33:39,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032419474 (current time:1694032419474). 2023-09-06T20:33:39,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:33:39,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:33:39,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b7bd1f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:39,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:39,482 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48978, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:33:39,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:33:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:33:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=115, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:33:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 115 2023-09-06T20:33:39,487 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:33:39,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=115 2023-09-06T20:33:39,488 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:33:39,490 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:33:39,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=115 2023-09-06T20:33:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=115 2023-09-06T20:33:39,896 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:33:39,899 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281}] 2023-09-06T20:33:39,901 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=116, ppid=115, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 2023-09-06T20:33:40,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=116 2023-09-06T20:33:40,052 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:33:40,052 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HRegion(2819): Flushing cf63a85c53279bc94811b50578f31281 2/2 column families, dataSize=1.50 KB heapSize=2.73 KB 2023-09-06T20:33:40,056 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/48f5b68b63f84de7a7c1f4f08993f64f is 171, key is trslm:hdfs://localhost:39327/backupUT\x00ns2:test-16940322964961/meta:log-roll-map/1694032410827/Put/seqid=0 2023-09-06T20:33:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=115 2023-09-06T20:33:40,460 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=717 B at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/48f5b68b63f84de7a7c1f4f08993f64f 2023-09-06T20:33:40,470 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/a2912410595b4f2d8ed499b1de1206c9 is 524, key is session:backup_1694032399062/session:context/1694032411656/Put/seqid=0 2023-09-06T20:33:40,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=115 2023-09-06T20:33:40,875 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=820 B at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/a2912410595b4f2d8ed499b1de1206c9 2023-09-06T20:33:40,881 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/48f5b68b63f84de7a7c1f4f08993f64f as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/48f5b68b63f84de7a7c1f4f08993f64f 2023-09-06T20:33:40,886 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/48f5b68b63f84de7a7c1f4f08993f64f, entries=6, sequenceid=94, filesize=5.8 K 2023-09-06T20:33:40,887 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/a2912410595b4f2d8ed499b1de1206c9 as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/a2912410595b4f2d8ed499b1de1206c9 2023-09-06T20:33:40,892 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/a2912410595b4f2d8ed499b1de1206c9, entries=2, sequenceid=94, filesize=5.5 K 2023-09-06T20:33:40,893 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HRegion(3022): Finished flush of dataSize ~1.50 KB/1537, heapSize ~2.70 KB/2760, currentSize=0 B/0 for cf63a85c53279bc94811b50578f31281 in 841ms, sequenceid=94, compaction requested=true 2023-09-06T20:33:40,893 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HRegion(2520): Flush status journal for cf63a85c53279bc94811b50578f31281: 2023-09-06T20:33:40,893 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. for snapshot_backup_system completed. 2023-09-06T20:33:40,893 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:33:40,893 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:33:40,893 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/0379b1e1f59e40d2ac2991b2ae6b86ce, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/dd7edbeade334fa7895ff079e0fc3472, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/78eec26cfa0242e48353b19a4a00b7d5, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/faffbcb72f8649bcad705936a3105fd4, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/01fb4a2eb68c4823807811190aa11f8b, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/b14bcdd382934779b0fa80ae042378aa, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/b7aca3aa47f24516bd2686bfd6b9193d, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/48f5b68b63f84de7a7c1f4f08993f64f] hfiles 2023-09-06T20:33:40,893 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (1/8): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/0379b1e1f59e40d2ac2991b2ae6b86ce for snapshot=snapshot_backup_system 2023-09-06T20:33:40,893 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (2/8): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/dd7edbeade334fa7895ff079e0fc3472 for snapshot=snapshot_backup_system 2023-09-06T20:33:40,893 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (3/8): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/78eec26cfa0242e48353b19a4a00b7d5 for snapshot=snapshot_backup_system 2023-09-06T20:33:40,893 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (4/8): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/faffbcb72f8649bcad705936a3105fd4 for snapshot=snapshot_backup_system 2023-09-06T20:33:40,893 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (5/8): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/01fb4a2eb68c4823807811190aa11f8b for snapshot=snapshot_backup_system 2023-09-06T20:33:40,893 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (6/8): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/b14bcdd382934779b0fa80ae042378aa for snapshot=snapshot_backup_system 2023-09-06T20:33:40,893 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (7/8): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/b7aca3aa47f24516bd2686bfd6b9193d for snapshot=snapshot_backup_system 2023-09-06T20:33:40,893 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (8/8): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/48f5b68b63f84de7a7c1f4f08993f64f for snapshot=snapshot_backup_system 2023-09-06T20:33:40,893 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/4ced064cdbd34bd6b594c76f049fa4ee, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/baca9a30045a4dabad9b3b78a8fd6c86, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/bcfde09dc46641bebf14d756f321d0af, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/2dc68dbd9f3749a1a2d9d7333fa4771d, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/6da826e7b47e41daa41e268fb4e951b2, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/849e36507ab34f85a508bffbc6e215d3, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/8bd33efd4a8e44d1b5b9f6a66a166990, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/13357027918440549abec030e3035318, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/a2912410595b4f2d8ed499b1de1206c9] hfiles 2023-09-06T20:33:40,893 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (1/9): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/4ced064cdbd34bd6b594c76f049fa4ee for snapshot=snapshot_backup_system 2023-09-06T20:33:40,893 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (2/9): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/baca9a30045a4dabad9b3b78a8fd6c86 for snapshot=snapshot_backup_system 2023-09-06T20:33:40,893 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (3/9): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/bcfde09dc46641bebf14d756f321d0af for snapshot=snapshot_backup_system 2023-09-06T20:33:40,893 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (4/9): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/2dc68dbd9f3749a1a2d9d7333fa4771d for snapshot=snapshot_backup_system 2023-09-06T20:33:40,893 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (5/9): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/6da826e7b47e41daa41e268fb4e951b2 for snapshot=snapshot_backup_system 2023-09-06T20:33:40,893 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (6/9): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/849e36507ab34f85a508bffbc6e215d3 for snapshot=snapshot_backup_system 2023-09-06T20:33:40,894 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (7/9): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/8bd33efd4a8e44d1b5b9f6a66a166990 for snapshot=snapshot_backup_system 2023-09-06T20:33:40,894 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (8/9): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/13357027918440549abec030e3035318 for snapshot=snapshot_backup_system 2023-09-06T20:33:40,894 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (9/9): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/a2912410595b4f2d8ed499b1de1206c9 for snapshot=snapshot_backup_system 2023-09-06T20:33:41,298 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:33:41,298 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=116 2023-09-06T20:33:41,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=116 2023-09-06T20:33:41,299 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region cf63a85c53279bc94811b50578f31281 2023-09-06T20:33:41,300 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=116, ppid=115, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 2023-09-06T20:33:41,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=116, resume processing ppid=115 2023-09-06T20:33:41,303 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:33:41,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=116, ppid=115, state=SUCCESS; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 in 1.4050 sec 2023-09-06T20:33:41,303 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:33:41,304 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:33:41,304 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:33:41,304 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:33:41,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=115 2023-09-06T20:33:41,716 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:33:41,728 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:33:41,728 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:33:41,729 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:33:41,729 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 115 2023-09-06T20:33:41,730 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=115, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2450 sec 2023-09-06T20:33:43,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=115 2023-09-06T20:33:43,593 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(115): Backup backup_1694032419452 started at 1694032423593. 2023-09-06T20:33:43,595 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(119): Backup session backup_1694032419452 has been started. 2023-09-06T20:33:43,600 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-09-06T20:33:43,600 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-09-06T20:33:43,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-09-06T20:33:43,601 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-09-06T20:33:43,601 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-09-06T20:33:43,602 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-09-06T20:33:43,602 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-09-06T20:33:43,602 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:33:43,603 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:43,603 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:33:43,603 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:33:43,604 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-09-06T20:33:43,604 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-09-06T20:33:43,604 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-09-06T20:33:43,605 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:33:43,605 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-09-06T20:33:43,605 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-09-06T20:33:43,605 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-09-06T20:33:43,605 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-09-06T20:33:43,605 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-09-06T20:33:43,606 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-09-06T20:33:43,606 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-09-06T20:33:43,606 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,36961,1694032287907' joining acquired barrier for procedure (rolllog) in zk 2023-09-06T20:33:43,608 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-09-06T20:33:43,608 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:43,608 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:33:43,608 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:33:43,608 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-09-06T20:33:43,608 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:33:43,609 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:33:43,609 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:33:43,609 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:43,610 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:33:43,610 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,36961,1694032287907' joining acquired barrier for procedure 'rolllog' on coordinator 2023-09-06T20:33:43,610 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-09-06T20:33:43,610 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@6f5085f3[Count = 0] remaining members to acquire global barrier 2023-09-06T20:33:43,610 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-09-06T20:33:43,611 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-09-06T20:33:43,611 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-09-06T20:33:43,611 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-09-06T20:33:43,612 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-09-06T20:33:43,612 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-09-06T20:33:43,612 DEBUG [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:43,612 INFO [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1694032401249 highest: 1694032401249 on jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:43,612 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta:.meta(num 1694032401230) roll requested 2023-09-06T20:33:43,623 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032423612.meta, exclude list is [], retry=0 2023-09-06T20:33:43,628 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK] 2023-09-06T20:33:43,630 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032401230.meta with entries=24, filesize=5.75 KB; new WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032423612.meta 2023-09-06T20:33:43,630 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK]] 2023-09-06T20:33:43,630 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032401230.meta is not closed yet, will try archiving it next time 2023-09-06T20:33:43,630 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36961%2C1694032287907:(num 1694032401249) roll requested 2023-09-06T20:33:43,641 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032423630, exclude list is [], retry=0 2023-09-06T20:33:43,644 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK] 2023-09-06T20:33:43,645 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032401249 with entries=15, filesize=4.08 KB; new WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032423630 2023-09-06T20:33:43,645 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK]] 2023-09-06T20:33:43,645 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032401249 is not closed yet, will try archiving it next time 2023-09-06T20:33:43,646 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032388895 to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/oldWALs/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032388895 2023-09-06T20:33:43,652 DEBUG [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-09-06T20:33:43,652 INFO [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1694032423630 on jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:43,659 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-09-06T20:33:43,659 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,36961,1694032287907' in zk 2023-09-06T20:33:43,663 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-09-06T20:33:43,663 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:33:43,663 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:43,663 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-09-06T20:33:43,663 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:33:43,665 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:33:43,666 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:33:43,666 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:33:43,666 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:33:43,666 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:43,667 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:33:43,667 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:33:43,667 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:43,668 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,36961,1694032287907': 2023-09-06T20:33:43,668 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,36961,1694032287907' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-09-06T20:33:43,668 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-09-06T20:33:43,668 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-09-06T20:33:43,668 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-09-06T20:33:43,668 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-09-06T20:33:43,668 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-09-06T20:33:43,669 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-09-06T20:33:43,669 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-09-06T20:33:43,669 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:33:43,669 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:33:43,669 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:33:43,670 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:33:43,670 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:33:43,670 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:33:43,670 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:33:43,670 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:33:43,670 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:33:43,671 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:33:43,671 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:43,671 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:33:43,671 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:33:43,672 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:43,679 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:33:43,679 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:33:43,679 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:33:43,679 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-09-06T20:33:43,679 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-09-06T20:33:43,679 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:33:43,679 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:33:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-09-06T20:33:43,783 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1694032423782_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } 2023-09-06T20:33:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032423783 (current time:1694032423783). 2023-09-06T20:33:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:33:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1694032423782_ns2_test-16940322964961 VERSION not specified, setting to 2 2023-09-06T20:33:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@340d80b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:43,791 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:47838, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:33:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:33:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:33:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=117, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1694032423782_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } 2023-09-06T20:33:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1694032423782_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 }, snapshot procedure id = 117 2023-09-06T20:33:43,795 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1694032423782_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:33:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=117 2023-09-06T20:33:43,796 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1694032423782_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:33:43,798 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1694032423782_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:33:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=117 2023-09-06T20:33:44,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=117 2023-09-06T20:33:44,204 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1694032423782_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:33:44,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE; SnapshotRegionProcedure cd8331e491339ba2021603869fc044dc}] 2023-09-06T20:33:44,207 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=118, ppid=117, state=RUNNABLE; SnapshotRegionProcedure cd8331e491339ba2021603869fc044dc 2023-09-06T20:33:44,263 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:33:44,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=118 2023-09-06T20:33:44,358 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc. 2023-09-06T20:33:44,358 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] regionserver.HRegion(2520): Flush status journal for cd8331e491339ba2021603869fc044dc: 2023-09-06T20:33:44,358 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc. for snapshot_1694032423782_ns2_test-16940322964961 completed. 2023-09-06T20:33:44,358 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc.' region-info for snapshot=snapshot_1694032423782_ns2_test-16940322964961 2023-09-06T20:33:44,358 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:33:44,358 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc/f/5f020f30827940489104cc2b9277b6d8_SeqId_4_] hfiles 2023-09-06T20:33:44,358 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc/f/5f020f30827940489104cc2b9277b6d8_SeqId_4_ for snapshot=snapshot_1694032423782_ns2_test-16940322964961 2023-09-06T20:33:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=117 2023-09-06T20:33:44,764 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc. 2023-09-06T20:33:44,764 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=118 2023-09-06T20:33:44,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=118 2023-09-06T20:33:44,764 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1694032423782_ns2_test-16940322964961 on region cd8331e491339ba2021603869fc044dc 2023-09-06T20:33:44,766 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=118, ppid=117, state=RUNNABLE; SnapshotRegionProcedure cd8331e491339ba2021603869fc044dc 2023-09-06T20:33:44,768 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=118, resume processing ppid=117 2023-09-06T20:33:44,768 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=118, ppid=117, state=SUCCESS; SnapshotRegionProcedure cd8331e491339ba2021603869fc044dc in 562 msec 2023-09-06T20:33:44,768 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1694032423782_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:33:44,768 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1694032423782_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:33:44,769 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1694032423782_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:33:44,769 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1694032423782_ns2_test-16940322964961 2023-09-06T20:33:44,770 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032423782_ns2_test-16940322964961 2023-09-06T20:33:44,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=117 2023-09-06T20:33:45,179 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1694032423782_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:33:45,185 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1694032423782_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:33:45,185 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032423782_ns2_test-16940322964961 to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032423782_ns2_test-16940322964961 2023-09-06T20:33:45,191 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1694032423782_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:33:45,191 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1694032423782_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 }, snapshot procedure id = 117 2023-09-06T20:33:45,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=117, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1694032423782_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } in 1.3980 sec 2023-09-06T20:33:45,747 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_table2 2023-09-06T20:33:45,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=117 2023-09-06T20:33:45,902 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1694032425901_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } 2023-09-06T20:33:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032425902 (current time:1694032425902). 2023-09-06T20:33:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:33:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1694032425901_ns3_test-16940322964962 VERSION not specified, setting to 2 2023-09-06T20:33:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@553a1b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:45,912 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:47844, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:33:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:33:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:33:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=119, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1694032425901_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } 2023-09-06T20:33:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1694032425901_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 }, snapshot procedure id = 119 2023-09-06T20:33:45,915 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1694032425901_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:33:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=119 2023-09-06T20:33:45,916 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1694032425901_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:33:45,918 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1694032425901_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:33:46,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=119 2023-09-06T20:33:46,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=119 2023-09-06T20:33:46,330 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1694032425901_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:33:46,330 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE; SnapshotRegionProcedure 685fcf26b1065c7882673d26a6d8d168}] 2023-09-06T20:33:46,332 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=120, ppid=119, state=RUNNABLE; SnapshotRegionProcedure 685fcf26b1065c7882673d26a6d8d168 2023-09-06T20:33:46,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=120 2023-09-06T20:33:46,484 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168. 2023-09-06T20:33:46,484 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] regionserver.HRegion(2520): Flush status journal for 685fcf26b1065c7882673d26a6d8d168: 2023-09-06T20:33:46,484 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168. for snapshot_1694032425901_ns3_test-16940322964962 completed. 2023-09-06T20:33:46,484 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168.' region-info for snapshot=snapshot_1694032425901_ns3_test-16940322964962 2023-09-06T20:33:46,484 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:33:46,484 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-09-06T20:33:46,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=119 2023-09-06T20:33:46,893 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168. 2023-09-06T20:33:46,893 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=120 2023-09-06T20:33:46,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=120 2023-09-06T20:33:46,894 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1694032425901_ns3_test-16940322964962 on region 685fcf26b1065c7882673d26a6d8d168 2023-09-06T20:33:46,895 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=120, ppid=119, state=RUNNABLE; SnapshotRegionProcedure 685fcf26b1065c7882673d26a6d8d168 2023-09-06T20:33:46,897 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=120, resume processing ppid=119 2023-09-06T20:33:46,897 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=120, ppid=119, state=SUCCESS; SnapshotRegionProcedure 685fcf26b1065c7882673d26a6d8d168 in 566 msec 2023-09-06T20:33:46,897 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1694032425901_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:33:46,898 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1694032425901_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:33:46,898 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1694032425901_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:33:46,898 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1694032425901_ns3_test-16940322964962 2023-09-06T20:33:46,899 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032425901_ns3_test-16940322964962 2023-09-06T20:33:47,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=119 2023-09-06T20:33:47,311 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1694032425901_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:33:47,316 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1694032425901_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:33:47,317 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032425901_ns3_test-16940322964962 to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032425901_ns3_test-16940322964962 2023-09-06T20:33:47,318 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1694032425901_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:33:47,318 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1694032425901_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 }, snapshot procedure id = 119 2023-09-06T20:33:47,320 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=119, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1694032425901_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } in 1.4050 sec 2023-09-06T20:33:48,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=119 2023-09-06T20:33:48,022 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1694032419452 2023-09-06T20:33:48,022 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-09-06T20:33:48,022 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-09-06T20:33:48,022 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1694032419452_ns3:test-16940322964962 2023-09-06T20:33:48,022 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1694032425901_ns3_test-16940322964962 to hdfs://localhost:39327/backupUT/backup_1694032419452/ns3/test-16940322964962/ 2023-09-06T20:33:48,023 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-09-06T20:33:48,046 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:33:48,046 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37419,1694032284546_780961049_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:39327/backupUT/backup_1694032419452/ns3/test-16940322964962, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:39327/backupUT/backup_1694032419452/ns3/test-16940322964962/.hbase-snapshot/.tmp/snapshot_1694032425901_ns3_test-16940322964962 2023-09-06T20:33:48,046 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1, snapshotDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032425901_ns3_test-16940322964962. 2023-09-06T20:33:48,052 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032425901_ns3_test-16940322964962 to hdfs://localhost:39327/backupUT/backup_1694032419452/ns3/test-16940322964962/.hbase-snapshot/.tmp/snapshot_1694032425901_ns3_test-16940322964962 2023-09-06T20:33:48,470 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:48,471 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:48,471 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:48,472 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:49,524 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop-8077517882737410535.jar 2023-09-06T20:33:49,525 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:49,525 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:49,525 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:49,526 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:49,526 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:49,526 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:49,527 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:49,527 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:49,527 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:49,528 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:33:49,528 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:33:49,529 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:33:49,529 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:33:49,530 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:33:49,530 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:33:49,530 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:33:49,531 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:33:49,531 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:33:49,531 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:33:49,532 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:33:49,532 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:33:49,533 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:33:49,533 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:33:49,534 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:33:49,534 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:33:49,534 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:33:49,535 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:33:49,535 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:33:49,585 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1694032425901_ns3_test-16940322964962' hfile list 2023-09-06T20:33:49,667 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032429625/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:33:49,667 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032429625/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:33:50,696 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-09-06T20:33:50,697 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-09-06T20:33:50,704 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1694032425901_ns3_test-16940322964962 2023-09-06T20:33:50,704 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1694032425901_ns3_test-16940322964962 finished. 2023-09-06T20:33:50,705 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1694032419452_ns2:test-16940322964961 2023-09-06T20:33:50,705 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1694032423782_ns2_test-16940322964961 to hdfs://localhost:39327/backupUT/backup_1694032419452/ns2/test-16940322964961/ 2023-09-06T20:33:50,705 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-09-06T20:33:50,729 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:33:50,729 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37419,1694032284546_-34732961_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:39327/backupUT/backup_1694032419452/ns2/test-16940322964961, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:39327/backupUT/backup_1694032419452/ns2/test-16940322964961/.hbase-snapshot/.tmp/snapshot_1694032423782_ns2_test-16940322964961 2023-09-06T20:33:50,729 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1, snapshotDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032423782_ns2_test-16940322964961. 2023-09-06T20:33:50,737 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032423782_ns2_test-16940322964961 to hdfs://localhost:39327/backupUT/backup_1694032419452/ns2/test-16940322964961/.hbase-snapshot/.tmp/snapshot_1694032423782_ns2_test-16940322964961 2023-09-06T20:33:51,182 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:51,183 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:51,184 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:51,184 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:51,264 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:33:52,187 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop-3210396331777715121.jar 2023-09-06T20:33:52,187 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:52,188 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:52,188 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:52,188 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:52,189 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:52,189 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:52,189 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:52,190 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:52,190 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:52,191 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:33:52,191 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:33:52,192 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:33:52,192 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:33:52,192 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:33:52,193 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:33:52,193 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:33:52,193 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:33:52,194 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:33:52,194 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:33:52,195 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:33:52,195 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:33:52,195 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:33:52,196 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:33:52,196 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:33:52,197 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:33:52,197 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:33:52,197 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:33:52,198 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:33:52,243 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1694032423782_ns2_test-16940322964961' hfile list 2023-09-06T20:33:52,245 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-09-06T20:33:52,314 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032432280/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:33:52,314 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032432280/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:33:52,367 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-09-06T20:33:52,415 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc/f/5f020f30827940489104cc2b9277b6d8_SeqId_4_ output=hdfs://localhost:39327/backupUT/backup_1694032419452/ns2/test-16940322964961/archive/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc/f/5f020f30827940489104cc2b9277b6d8_SeqId_4_ 2023-09-06T20:33:52,415 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 4.160M/sec 2023-09-06T20:33:53,345 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-09-06T20:33:53,346 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-09-06T20:33:53,352 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1694032423782_ns2_test-16940322964961 2023-09-06T20:33:53,352 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1694032423782_ns2_test-16940322964961 finished. 2023-09-06T20:33:53,352 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1014): ns3:test-16940322964962 2023-09-06T20:33:53,352 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1014): ns2:test-16940322964961 2023-09-06T20:33:53,358 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1694032419452 2023-09-06T20:33:53,359 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-09-06T20:33:53,765 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:39327/backupUT/backup_1694032419452/.backup.manifest 2023-09-06T20:33:53,765 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1694032419452 2023-09-06T20:33:53,765 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-09-06T20:33:54,172 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:39327/backupUT/backup_1694032419452/.backup.manifest 2023-09-06T20:33:54,172 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(358): Backup backup_1694032419452 finished: type=FULL,tablelist=ns3:test-16940322964962;ns2:test-16940322964961,targetRootDir=hdfs://localhost:39327/backupUT,startts=1694032423593,completets=1694032433358,bytescopied=0 2023-09-06T20:33:54,172 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-09-06T20:33:54,172 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1694032425901_ns3_test-16940322964962 2023-09-06T20:33:54,173 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1694032425901_ns3_test-16940322964962" 2023-09-06T20:33:54,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1694032425901_ns3_test-16940322964962 2023-09-06T20:33:54,179 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1694032425901_ns3_test-16940322964962 for backup backup_1694032419452 succeeded. 2023-09-06T20:33:54,179 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1694032423782_ns2_test-16940322964961 2023-09-06T20:33:54,180 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1694032423782_ns2_test-16940322964961" 2023-09-06T20:33:54,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1694032423782_ns2_test-16940322964961 2023-09-06T20:33:54,183 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1694032423782_ns2_test-16940322964961 for backup backup_1694032419452 succeeded. 2023-09-06T20:33:54,183 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:33:54,187 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:33:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:33:54,189 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:33:54,190 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:33:54,195 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(376): Backup backup_1694032419452 completed. 2023-09-06T20:33:54,195 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:33:54,204 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] backup.RestoreDriver(101): Found -overwrite option in restore command, will overwrite to existing table if any in the restore target 2023-09-06T20:33:54,209 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a4b259a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:54,210 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:54,212 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:34092, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:33:54,215 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:39327/backupUT/backup_1694032419452/.backup.manifest 2023-09-06T20:33:54,217 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032419452/.backup.manifest 2023-09-06T20:33:54,217 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:39327/backupUT/backup_1694032419452/.backup.manifest 2023-09-06T20:33:54,219 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032419452/.backup.manifest 2023-09-06T20:33:54,219 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:39327/backupUT/backup_1694032419452/.backup.manifest 2023-09-06T20:33:54,221 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032419452/.backup.manifest 2023-09-06T20:33:54,221 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(148): Restoring 'ns2:test-16940322964961' to 'ns2:test-16940322964961' from full backup image hdfs://localhost:39327/backupUT/backup_1694032419452/ns2/test-16940322964961 2023-09-06T20:33:54,227 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(476): Truncating exising target table 'ns2:test-16940322964961', preserving region splits 2023-09-06T20:33:54,227 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:33:54,229 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:58104, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:33:54,229 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable ns2:test-16940322964961 2023-09-06T20:33:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=121, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns2:test-16940322964961 2023-09-06T20:33:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=121 2023-09-06T20:33:54,233 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16940322964961","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032434233"}]},"ts":"1694032434233"} 2023-09-06T20:33:54,234 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16940322964961, state=DISABLING in hbase:meta 2023-09-06T20:33:54,236 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set ns2:test-16940322964961 to state=DISABLING 2023-09-06T20:33:54,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:test-16940322964961, region=cd8331e491339ba2021603869fc044dc, UNASSIGN}] 2023-09-06T20:33:54,239 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=122, ppid=121, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:test-16940322964961, region=cd8331e491339ba2021603869fc044dc, UNASSIGN 2023-09-06T20:33:54,240 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=122 updating hbase:meta row=cd8331e491339ba2021603869fc044dc, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:54,241 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=123, ppid=122, state=RUNNABLE; CloseRegionProcedure cd8331e491339ba2021603869fc044dc, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:33:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=121 2023-09-06T20:33:54,393 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] handler.UnassignRegionHandler(119): Close cd8331e491339ba2021603869fc044dc 2023-09-06T20:33:54,393 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1668): Closing cd8331e491339ba2021603869fc044dc, disabling compactions & flushes 2023-09-06T20:33:54,393 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1690): Closing region ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc. 2023-09-06T20:33:54,393 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc. 2023-09-06T20:33:54,393 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc. after waiting 0 ms 2023-09-06T20:33:54,393 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc. 2023-09-06T20:33:54,400 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2023-09-06T20:33:54,401 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:33:54,401 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1909): Closed ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc. 2023-09-06T20:33:54,401 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1622): Region close journal for cd8331e491339ba2021603869fc044dc: 2023-09-06T20:33:54,403 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] handler.UnassignRegionHandler(163): Closed cd8331e491339ba2021603869fc044dc 2023-09-06T20:33:54,403 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=122 updating hbase:meta row=cd8331e491339ba2021603869fc044dc, regionState=CLOSED 2023-09-06T20:33:54,406 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=123, resume processing ppid=122 2023-09-06T20:33:54,406 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=123, ppid=122, state=SUCCESS; CloseRegionProcedure cd8331e491339ba2021603869fc044dc, server=jenkins-hbase3.apache.org,36961,1694032287907 in 164 msec 2023-09-06T20:33:54,408 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=122, resume processing ppid=121 2023-09-06T20:33:54,408 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=122, ppid=121, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16940322964961, region=cd8331e491339ba2021603869fc044dc, UNASSIGN in 170 msec 2023-09-06T20:33:54,408 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16940322964961","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032434408"}]},"ts":"1694032434408"} 2023-09-06T20:33:54,409 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16940322964961, state=DISABLED in hbase:meta 2023-09-06T20:33:54,411 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set ns2:test-16940322964961 to state=DISABLED 2023-09-06T20:33:54,413 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=121, state=SUCCESS; DisableTableProcedure table=ns2:test-16940322964961 in 182 msec 2023-09-06T20:33:54,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=121 2023-09-06T20:33:54,535 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$6(2480): Client=jenkins//172.31.12.81 truncate ns2:test-16940322964961 2023-09-06T20:33:54,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=124, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=ns2:test-16940322964961 preserveSplits=true) 2023-09-06T20:33:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=124 2023-09-06T20:33:54,538 DEBUG [PEWorker-2 {}] procedure.TruncateTableProcedure(87): waiting for 'ns2:test-16940322964961' regions in transition 2023-09-06T20:33:54,541 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc 2023-09-06T20:33:54,543 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc/f, FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc/recovered.edits] 2023-09-06T20:33:54,547 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc/f/5f020f30827940489104cc2b9277b6d8_SeqId_4_ to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/archive/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc/f/5f020f30827940489104cc2b9277b6d8_SeqId_4_ 2023-09-06T20:33:54,551 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc/recovered.edits/9.seqid to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/archive/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc/recovered.edits/9.seqid 2023-09-06T20:33:54,551 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc 2023-09-06T20:33:54,551 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(293): Archived ns2:test-16940322964961 regions 2023-09-06T20:33:54,556 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns2:test-16940322964961 from hbase:meta 2023-09-06T20:33:54,558 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(387): Removing 'ns2:test-16940322964961' descriptor. 2023-09-06T20:33:54,558 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(377): Removing 'ns2:test-16940322964961' from region states. 2023-09-06T20:33:54,558 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1694032434558"}]},"ts":"9223372036854775807"} 2023-09-06T20:33:54,560 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-06T20:33:54,560 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => cd8331e491339ba2021603869fc044dc, NAME => 'ns2:test-16940322964961,,1694032318770.cd8331e491339ba2021603869fc044dc.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:33:54,560 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(381): Marking 'ns2:test-16940322964961' as deleted. 2023-09-06T20:33:54,560 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:test-16940322964961","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1694032434560"}]},"ts":"9223372036854775807"} 2023-09-06T20:33:54,561 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1678): Deleted table ns2:test-16940322964961 state from META 2023-09-06T20:33:54,572 INFO [RegionOpenAndInit-ns2:test-16940322964961-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => e70b91c8ad9e774e6ad3f358efadcaf1, NAME => 'ns2:test-16940322964961,,1694032434553.e70b91c8ad9e774e6ad3f358efadcaf1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16940322964961', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:33:54,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=124 2023-09-06T20:33:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=124 2023-09-06T20:33:54,978 DEBUG [RegionOpenAndInit-ns2:test-16940322964961-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16940322964961,,1694032434553.e70b91c8ad9e774e6ad3f358efadcaf1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:33:54,978 DEBUG [RegionOpenAndInit-ns2:test-16940322964961-pool-0 {}] regionserver.HRegion(1668): Closing e70b91c8ad9e774e6ad3f358efadcaf1, disabling compactions & flushes 2023-09-06T20:33:54,978 INFO [RegionOpenAndInit-ns2:test-16940322964961-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16940322964961,,1694032434553.e70b91c8ad9e774e6ad3f358efadcaf1. 2023-09-06T20:33:54,978 DEBUG [RegionOpenAndInit-ns2:test-16940322964961-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16940322964961,,1694032434553.e70b91c8ad9e774e6ad3f358efadcaf1. 2023-09-06T20:33:54,978 DEBUG [RegionOpenAndInit-ns2:test-16940322964961-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16940322964961,,1694032434553.e70b91c8ad9e774e6ad3f358efadcaf1. after waiting 0 ms 2023-09-06T20:33:54,978 DEBUG [RegionOpenAndInit-ns2:test-16940322964961-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16940322964961,,1694032434553.e70b91c8ad9e774e6ad3f358efadcaf1. 2023-09-06T20:33:54,978 INFO [RegionOpenAndInit-ns2:test-16940322964961-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16940322964961,,1694032434553.e70b91c8ad9e774e6ad3f358efadcaf1. 2023-09-06T20:33:54,978 DEBUG [RegionOpenAndInit-ns2:test-16940322964961-pool-0 {}] regionserver.HRegion(1622): Region close journal for e70b91c8ad9e774e6ad3f358efadcaf1: 2023-09-06T20:33:54,979 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16940322964961,,1694032434553.e70b91c8ad9e774e6ad3f358efadcaf1.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1694032434979"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032434979"}]},"ts":"1694032434979"} 2023-09-06T20:33:54,980 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:33:54,981 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16940322964961","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032434981"}]},"ts":"1694032434981"} 2023-09-06T20:33:54,982 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16940322964961, state=ENABLING in hbase:meta 2023-09-06T20:33:54,987 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16940322964961, region=e70b91c8ad9e774e6ad3f358efadcaf1, ASSIGN}] 2023-09-06T20:33:54,988 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16940322964961, region=e70b91c8ad9e774e6ad3f358efadcaf1, ASSIGN 2023-09-06T20:33:54,989 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16940322964961, region=e70b91c8ad9e774e6ad3f358efadcaf1, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36961,1694032287907; forceNewPlan=false, retain=false 2023-09-06T20:33:55,141 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=125 updating hbase:meta row=e70b91c8ad9e774e6ad3f358efadcaf1, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:55,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=124 2023-09-06T20:33:55,143 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE; OpenRegionProcedure e70b91c8ad9e774e6ad3f358efadcaf1, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:33:55,298 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(130): Open ns2:test-16940322964961,,1694032434553.e70b91c8ad9e774e6ad3f358efadcaf1. 2023-09-06T20:33:55,298 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7254): Opening region: {ENCODED => e70b91c8ad9e774e6ad3f358efadcaf1, NAME => 'ns2:test-16940322964961,,1694032434553.e70b91c8ad9e774e6ad3f358efadcaf1.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:33:55,299 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:33:55,299 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16940322964961 e70b91c8ad9e774e6ad3f358efadcaf1 2023-09-06T20:33:55,299 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(888): Instantiated ns2:test-16940322964961,,1694032434553.e70b91c8ad9e774e6ad3f358efadcaf1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:33:55,299 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7296): checking encryption for e70b91c8ad9e774e6ad3f358efadcaf1 2023-09-06T20:33:55,299 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7299): checking classloading for e70b91c8ad9e774e6ad3f358efadcaf1 2023-09-06T20:33:55,301 INFO [StoreOpener-e70b91c8ad9e774e6ad3f358efadcaf1-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region e70b91c8ad9e774e6ad3f358efadcaf1 2023-09-06T20:33:55,302 INFO [StoreOpener-e70b91c8ad9e774e6ad3f358efadcaf1-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e70b91c8ad9e774e6ad3f358efadcaf1 columnFamilyName f 2023-09-06T20:33:55,302 DEBUG [StoreOpener-e70b91c8ad9e774e6ad3f358efadcaf1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:33:55,303 INFO [StoreOpener-e70b91c8ad9e774e6ad3f358efadcaf1-1 {}] regionserver.HStore(324): Store=e70b91c8ad9e774e6ad3f358efadcaf1/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:33:55,304 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/e70b91c8ad9e774e6ad3f358efadcaf1 2023-09-06T20:33:55,304 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/e70b91c8ad9e774e6ad3f358efadcaf1 2023-09-06T20:33:55,307 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1076): writing seq id for e70b91c8ad9e774e6ad3f358efadcaf1 2023-09-06T20:33:55,309 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/e70b91c8ad9e774e6ad3f358efadcaf1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:33:55,309 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1093): Opened e70b91c8ad9e774e6ad3f358efadcaf1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9623005120, jitterRate=-0.10378780961036682}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:33:55,310 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(992): Region open journal for e70b91c8ad9e774e6ad3f358efadcaf1: 2023-09-06T20:33:55,310 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:test-16940322964961,,1694032434553.e70b91c8ad9e774e6ad3f358efadcaf1., pid=126, masterSystemTime=1694032435295 2023-09-06T20:33:55,312 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:test-16940322964961,,1694032434553.e70b91c8ad9e774e6ad3f358efadcaf1. 2023-09-06T20:33:55,312 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(158): Opened ns2:test-16940322964961,,1694032434553.e70b91c8ad9e774e6ad3f358efadcaf1. 2023-09-06T20:33:55,312 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=125 updating hbase:meta row=e70b91c8ad9e774e6ad3f358efadcaf1, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:55,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=126, resume processing ppid=125 2023-09-06T20:33:55,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=126, ppid=125, state=SUCCESS; OpenRegionProcedure e70b91c8ad9e774e6ad3f358efadcaf1, server=jenkins-hbase3.apache.org,36961,1694032287907 in 170 msec 2023-09-06T20:33:55,316 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=125, resume processing ppid=124 2023-09-06T20:33:55,316 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=125, ppid=124, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16940322964961, region=e70b91c8ad9e774e6ad3f358efadcaf1, ASSIGN in 328 msec 2023-09-06T20:33:55,316 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16940322964961","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032435316"}]},"ts":"1694032435316"} 2023-09-06T20:33:55,317 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16940322964961, state=ENABLED in hbase:meta 2023-09-06T20:33:55,319 DEBUG [PEWorker-3 {}] procedure.TruncateTableProcedure(145): truncate 'ns2:test-16940322964961' completed 2023-09-06T20:33:55,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=124, state=SUCCESS; TruncateTableProcedure (table=ns2:test-16940322964961 preserveSplits=true) in 783 msec 2023-09-06T20:33:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=124 2023-09-06T20:33:55,649 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:39327/backupUT/backup_1694032419452/ns2/test-16940322964961/archive/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc from hbase tables ns2:test-16940322964961 to tables ns2:test-16940322964961 2023-09-06T20:33:55,649 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-16940322964961 into ns2:test-16940322964961 2023-09-06T20:33:55,650 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-test-16940322964961-1694032435649 from hdfs://localhost:39327/backupUT/backup_1694032419452/ns2/test-16940322964961/archive/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc 2023-09-06T20:33:55,655 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@754401b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:55,656 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:33:55,657 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:58114, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:33:55,658 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-09-06T20:33:55,658 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16940322964961 2023-09-06T20:33:55,660 DEBUG [hconnection-0x1adf2f5e-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:55,661 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:34096, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:33:55,662 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16940322964961: 2023-09-06T20:33:55,662 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-09-06T20:33:55,663 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/partitions_6e0bb028-77bf-4a83-baaf-558c6138d9c1 2023-09-06T20:33:55,669 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:55,669 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:55,670 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:55,670 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:56,706 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop-2817978878241782763.jar 2023-09-06T20:33:56,706 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:56,707 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:56,707 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:56,707 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:56,708 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:56,708 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:56,708 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:56,709 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:56,709 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:56,710 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:33:56,710 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:33:56,710 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:33:56,711 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:33:56,711 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:33:56,712 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:33:56,712 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:33:56,712 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:33:56,713 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:33:56,713 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:33:56,713 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:33:56,714 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:33:56,714 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:56,715 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:56,715 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:56,715 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:56,715 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:56,716 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:33:56,716 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:33:56,716 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:test-16940322964961 2023-09-06T20:33:56,717 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:33:56,717 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-09-06T20:33:56,717 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-09-06T20:33:56,718 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:33:56,728 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-09-06T20:33:56,805 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032436770/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:33:56,805 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032436770/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:33:56,856 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:39327/backupUT/backup_1694032419452/ns2/test-16940322964961/archive/data/ns2/test-16940322964961/cd8331e491339ba2021603869fc044dc/f/5f020f30827940489104cc2b9277b6d8_SeqId_4_ 2023-09-06T20:33:56,860 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-09-06T20:33:56,880 DEBUG [pool-440-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74ffd185, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:56,881 DEBUG [hconnection-0x4ebd7214-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:56,883 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:34112, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:33:56,886 DEBUG [pool-440-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:33:56,886 DEBUG [pool-440-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-09-06T20:33:56,886 DEBUG [pool-440-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-09-06T20:33:56,892 DEBUG [pool-440-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-test-16940322964961-1694032435649/_temporary/0/_temporary/attempt_local1110817312_0019_r_000000_0/f/e96cf968b8d94144a394bf383e95ef36 is 37, key is row10/f:q1/1694032300351/Put/seqid=0 2023-09-06T20:33:56,893 WARN [Thread-1917 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:36961] Specified favored nodes: [jenkins-hbase3.apache.org:36961] 2023-09-06T20:33:57,859 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-test-16940322964961-1694032435649 2023-09-06T20:33:57,866 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61bf6d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:57,867 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:57,869 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:34124, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:33:57,875 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-test-16940322964961-1694032435649/_SUCCESS 2023-09-06T20:33:57,879 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:33:57,881 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:58126, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:33:57,895 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-test-16940322964961-1694032435649/f/e96cf968b8d94144a394bf383e95ef36 first=Optional[row0] last=Optional[row98] 2023-09-06T20:33:57,897 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=ns2:test-16940322964961,,1694032434553.e70b91c8ad9e774e6ad3f358efadcaf1., hostname=jenkins-hbase3.apache.org,36961,1694032287907, seqNum=2 for row with hfile group [{f,hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-test-16940322964961-1694032435649/f/e96cf968b8d94144a394bf383e95ef36}] 2023-09-06T20:33:57,905 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-test-16940322964961-1694032435649/f/e96cf968b8d94144a394bf383e95ef36 for inclusion in e70b91c8ad9e774e6ad3f358efadcaf1/f 2023-09-06T20:33:57,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-09-06T20:33:57,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(625): Region bounds: first= last= 2023-09-06T20:33:57,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HRegion(2520): Flush status journal for e70b91c8ad9e774e6ad3f358efadcaf1: 2023-09-06T20:33:57,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-test-16940322964961-1694032435649/f/e96cf968b8d94144a394bf383e95ef36 to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__ns2_test-16940322964961__nadbdmj2kjvkd1eghhcop7dd783t309j69t3amfa3cplp9qoqniufaa7sv0o53gf/f/e96cf968b8d94144a394bf383e95ef36 2023-09-06T20:33:57,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__ns2_test-16940322964961__nadbdmj2kjvkd1eghhcop7dd783t309j69t3amfa3cplp9qoqniufaa7sv0o53gf/f/e96cf968b8d94144a394bf383e95ef36 as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/e70b91c8ad9e774e6ad3f358efadcaf1/f/033098d0c12f4e7492b697713877c54c_SeqId_4_ 2023-09-06T20:33:57,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bd2490c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:57,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:33:57,933 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:58134, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:33:57,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:57,938 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:34134, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:33:57,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] impl.BackupSystemTable(408): write bulk load descriptor to backup ns2:test-16940322964961 with 1 entries 2023-09-06T20:33:57,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] impl.BackupSystemTable(1582): writing raw bulk path hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/e70b91c8ad9e774e6ad3f358efadcaf1/f/033098d0c12f4e7492b697713877c54c_SeqId_4_ for ns2:test-16940322964961 e70b91c8ad9e774e6ad3f358efadcaf1 2023-09-06T20:33:57,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] impl.BackupSystemTable(415): written 1 rows for bulk load of ns2:test-16940322964961 2023-09-06T20:33:57,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:33:57,964 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__ns2_test-16940322964961__nadbdmj2kjvkd1eghhcop7dd783t309j69t3amfa3cplp9qoqniufaa7sv0o53gf/f/e96cf968b8d94144a394bf383e95ef36 into e70b91c8ad9e774e6ad3f358efadcaf1/f as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/e70b91c8ad9e774e6ad3f358efadcaf1/f/033098d0c12f4e7492b697713877c54c_SeqId_4_ - updating store file list. 2023-09-06T20:33:57,968 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/e70b91c8ad9e774e6ad3f358efadcaf1/f/033098d0c12f4e7492b697713877c54c_SeqId_4_ into e70b91c8ad9e774e6ad3f358efadcaf1/f 2023-09-06T20:33:57,968 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__ns2_test-16940322964961__nadbdmj2kjvkd1eghhcop7dd783t309j69t3amfa3cplp9qoqniufaa7sv0o53gf/f/e96cf968b8d94144a394bf383e95ef36 into e70b91c8ad9e774e6ad3f358efadcaf1/f (new location: hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/e70b91c8ad9e774e6ad3f358efadcaf1/f/033098d0c12f4e7492b697713877c54c_SeqId_4_) 2023-09-06T20:33:57,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__ns2_test-16940322964961__nadbdmj2kjvkd1eghhcop7dd783t309j69t3amfa3cplp9qoqniufaa7sv0o53gf/f/e96cf968b8d94144a394bf383e95ef36 2023-09-06T20:33:57,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e110742, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:57,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:33:57,978 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:58146, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-09-06T20:33:57,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:57,984 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:34146, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-09-06T20:33:57,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] impl.BackupSystemTable(388): write bulk load descriptor to backup ns2:test-16940322964961 with 1 entries 2023-09-06T20:33:57,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] impl.BackupSystemTable(1503): writing done bulk path hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/e70b91c8ad9e774e6ad3f358efadcaf1/f/033098d0c12f4e7492b697713877c54c_SeqId_4_ for ns2:test-16940322964961 e70b91c8ad9e774e6ad3f358efadcaf1 2023-09-06T20:33:57,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] impl.BackupSystemTable(394): written 1 rows for bulk load of ns2:test-16940322964961 2023-09-06T20:33:57,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:33:58,000 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:33:58,000 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-09-06T20:33:58,000 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-09-06T20:33:58,000 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(233): Backup: backup_1694032419452 hdfs://localhost:39327/backupUT/backup_1694032419452/ns2/test-16940322964961/ 2023-09-06T20:33:58,001 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:39327/backupUT/backup_1694032419452/.backup.manifest 2023-09-06T20:33:58,003 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032419452/.backup.manifest 2023-09-06T20:33:58,003 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(148): Restoring 'ns3:test-16940322964962' to 'ns3:test-16940322964962' from full backup image hdfs://localhost:39327/backupUT/backup_1694032419452/ns3/test-16940322964962 2023-09-06T20:33:58,007 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(101): Folder tableArchivePath: hdfs://localhost:39327/backupUT/backup_1694032419452/ns3/test-16940322964962/archive/data/ns3/test-16940322964962 does not exists 2023-09-06T20:33:58,007 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(326): find table descriptor but no archive dir for table ns3:test-16940322964962, will only create table 2023-09-06T20:33:58,008 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(476): Truncating exising target table 'ns3:test-16940322964962', preserving region splits 2023-09-06T20:33:58,010 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable ns3:test-16940322964962 2023-09-06T20:33:58,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=127, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns3:test-16940322964962 2023-09-06T20:33:58,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=127 2023-09-06T20:33:58,013 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16940322964962","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032438013"}]},"ts":"1694032438013"} 2023-09-06T20:33:58,014 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16940322964962, state=DISABLING in hbase:meta 2023-09-06T20:33:58,015 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set ns3:test-16940322964962 to state=DISABLING 2023-09-06T20:33:58,016 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:test-16940322964962, region=685fcf26b1065c7882673d26a6d8d168, UNASSIGN}] 2023-09-06T20:33:58,017 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=128, ppid=127, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:test-16940322964962, region=685fcf26b1065c7882673d26a6d8d168, UNASSIGN 2023-09-06T20:33:58,018 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=128 updating hbase:meta row=685fcf26b1065c7882673d26a6d8d168, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:58,019 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE; CloseRegionProcedure 685fcf26b1065c7882673d26a6d8d168, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:33:58,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=127 2023-09-06T20:33:58,170 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] handler.UnassignRegionHandler(119): Close 685fcf26b1065c7882673d26a6d8d168 2023-09-06T20:33:58,170 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1668): Closing 685fcf26b1065c7882673d26a6d8d168, disabling compactions & flushes 2023-09-06T20:33:58,170 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1690): Closing region ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168. 2023-09-06T20:33:58,170 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168. 2023-09-06T20:33:58,170 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168. after waiting 0 ms 2023-09-06T20:33:58,170 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168. 2023-09-06T20:33:58,173 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/test-16940322964962/685fcf26b1065c7882673d26a6d8d168/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=1 2023-09-06T20:33:58,174 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:33:58,175 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1909): Closed ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168. 2023-09-06T20:33:58,175 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1622): Region close journal for 685fcf26b1065c7882673d26a6d8d168: 2023-09-06T20:33:58,176 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] handler.UnassignRegionHandler(163): Closed 685fcf26b1065c7882673d26a6d8d168 2023-09-06T20:33:58,176 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=128 updating hbase:meta row=685fcf26b1065c7882673d26a6d8d168, regionState=CLOSED 2023-09-06T20:33:58,178 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=129, resume processing ppid=128 2023-09-06T20:33:58,178 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=129, ppid=128, state=SUCCESS; CloseRegionProcedure 685fcf26b1065c7882673d26a6d8d168, server=jenkins-hbase3.apache.org,36961,1694032287907 in 158 msec 2023-09-06T20:33:58,179 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=128, resume processing ppid=127 2023-09-06T20:33:58,179 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=128, ppid=127, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16940322964962, region=685fcf26b1065c7882673d26a6d8d168, UNASSIGN in 162 msec 2023-09-06T20:33:58,180 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16940322964962","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032438180"}]},"ts":"1694032438180"} 2023-09-06T20:33:58,181 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16940322964962, state=DISABLED in hbase:meta 2023-09-06T20:33:58,182 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(305): Set ns3:test-16940322964962 to state=DISABLED 2023-09-06T20:33:58,183 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=127, state=SUCCESS; DisableTableProcedure table=ns3:test-16940322964962 in 172 msec 2023-09-06T20:33:58,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=127 2023-09-06T20:33:58,315 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$6(2480): Client=jenkins//172.31.12.81 truncate ns3:test-16940322964962 2023-09-06T20:33:58,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=130, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=ns3:test-16940322964962 preserveSplits=true) 2023-09-06T20:33:58,317 DEBUG [PEWorker-5 {}] procedure.TruncateTableProcedure(87): waiting for 'ns3:test-16940322964962' regions in transition 2023-09-06T20:33:58,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=130 2023-09-06T20:33:58,320 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/test-16940322964962/685fcf26b1065c7882673d26a6d8d168 2023-09-06T20:33:58,321 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/test-16940322964962/685fcf26b1065c7882673d26a6d8d168/f, FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/test-16940322964962/685fcf26b1065c7882673d26a6d8d168/recovered.edits] 2023-09-06T20:33:58,325 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/test-16940322964962/685fcf26b1065c7882673d26a6d8d168/recovered.edits/7.seqid to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/archive/data/ns3/test-16940322964962/685fcf26b1065c7882673d26a6d8d168/recovered.edits/7.seqid 2023-09-06T20:33:58,326 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/test-16940322964962/685fcf26b1065c7882673d26a6d8d168 2023-09-06T20:33:58,326 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(293): Archived ns3:test-16940322964962 regions 2023-09-06T20:33:58,329 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns3:test-16940322964962 from hbase:meta 2023-09-06T20:33:58,331 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(387): Removing 'ns3:test-16940322964962' descriptor. 2023-09-06T20:33:58,331 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(377): Removing 'ns3:test-16940322964962' from region states. 2023-09-06T20:33:58,331 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1694032438331"}]},"ts":"9223372036854775807"} 2023-09-06T20:33:58,333 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-06T20:33:58,333 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 685fcf26b1065c7882673d26a6d8d168, NAME => 'ns3:test-16940322964962,,1694032324181.685fcf26b1065c7882673d26a6d8d168.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:33:58,333 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(381): Marking 'ns3:test-16940322964962' as deleted. 2023-09-06T20:33:58,333 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:test-16940322964962","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1694032438333"}]},"ts":"9223372036854775807"} 2023-09-06T20:33:58,335 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1678): Deleted table ns3:test-16940322964962 state from META 2023-09-06T20:33:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=130 2023-09-06T20:33:58,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=130 2023-09-06T20:33:58,743 INFO [RegionOpenAndInit-ns3:test-16940322964962-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 895b7c16371d966da7eb221de358652c, NAME => 'ns3:test-16940322964962,,1694032438327.895b7c16371d966da7eb221de358652c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16940322964962', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:33:58,748 DEBUG [RegionOpenAndInit-ns3:test-16940322964962-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16940322964962,,1694032438327.895b7c16371d966da7eb221de358652c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:33:58,748 DEBUG [RegionOpenAndInit-ns3:test-16940322964962-pool-0 {}] regionserver.HRegion(1668): Closing 895b7c16371d966da7eb221de358652c, disabling compactions & flushes 2023-09-06T20:33:58,748 INFO [RegionOpenAndInit-ns3:test-16940322964962-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16940322964962,,1694032438327.895b7c16371d966da7eb221de358652c. 2023-09-06T20:33:58,748 DEBUG [RegionOpenAndInit-ns3:test-16940322964962-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16940322964962,,1694032438327.895b7c16371d966da7eb221de358652c. 2023-09-06T20:33:58,748 DEBUG [RegionOpenAndInit-ns3:test-16940322964962-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16940322964962,,1694032438327.895b7c16371d966da7eb221de358652c. after waiting 0 ms 2023-09-06T20:33:58,748 DEBUG [RegionOpenAndInit-ns3:test-16940322964962-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16940322964962,,1694032438327.895b7c16371d966da7eb221de358652c. 2023-09-06T20:33:58,748 INFO [RegionOpenAndInit-ns3:test-16940322964962-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16940322964962,,1694032438327.895b7c16371d966da7eb221de358652c. 2023-09-06T20:33:58,748 DEBUG [RegionOpenAndInit-ns3:test-16940322964962-pool-0 {}] regionserver.HRegion(1622): Region close journal for 895b7c16371d966da7eb221de358652c: 2023-09-06T20:33:58,749 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16940322964962,,1694032438327.895b7c16371d966da7eb221de358652c.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1694032438749"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032438749"}]},"ts":"1694032438749"} 2023-09-06T20:33:58,751 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:33:58,751 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16940322964962","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032438751"}]},"ts":"1694032438751"} 2023-09-06T20:33:58,752 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16940322964962, state=ENABLING in hbase:meta 2023-09-06T20:33:58,757 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16940322964962, region=895b7c16371d966da7eb221de358652c, ASSIGN}] 2023-09-06T20:33:58,759 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16940322964962, region=895b7c16371d966da7eb221de358652c, ASSIGN 2023-09-06T20:33:58,760 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16940322964962, region=895b7c16371d966da7eb221de358652c, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36961,1694032287907; forceNewPlan=false, retain=false 2023-09-06T20:33:58,911 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=131 updating hbase:meta row=895b7c16371d966da7eb221de358652c, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:58,913 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE; OpenRegionProcedure 895b7c16371d966da7eb221de358652c, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:33:58,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=130 2023-09-06T20:33:59,068 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] handler.AssignRegionHandler(130): Open ns3:test-16940322964962,,1694032438327.895b7c16371d966da7eb221de358652c. 2023-09-06T20:33:59,068 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7254): Opening region: {ENCODED => 895b7c16371d966da7eb221de358652c, NAME => 'ns3:test-16940322964962,,1694032438327.895b7c16371d966da7eb221de358652c.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:33:59,069 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:33:59,069 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16940322964962 895b7c16371d966da7eb221de358652c 2023-09-06T20:33:59,069 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(888): Instantiated ns3:test-16940322964962,,1694032438327.895b7c16371d966da7eb221de358652c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:33:59,069 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7296): checking encryption for 895b7c16371d966da7eb221de358652c 2023-09-06T20:33:59,069 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7299): checking classloading for 895b7c16371d966da7eb221de358652c 2023-09-06T20:33:59,070 INFO [StoreOpener-895b7c16371d966da7eb221de358652c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 895b7c16371d966da7eb221de358652c 2023-09-06T20:33:59,071 INFO [StoreOpener-895b7c16371d966da7eb221de358652c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 895b7c16371d966da7eb221de358652c columnFamilyName f 2023-09-06T20:33:59,071 DEBUG [StoreOpener-895b7c16371d966da7eb221de358652c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:33:59,072 INFO [StoreOpener-895b7c16371d966da7eb221de358652c-1 {}] regionserver.HStore(324): Store=895b7c16371d966da7eb221de358652c/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:33:59,073 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/test-16940322964962/895b7c16371d966da7eb221de358652c 2023-09-06T20:33:59,073 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/test-16940322964962/895b7c16371d966da7eb221de358652c 2023-09-06T20:33:59,075 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1076): writing seq id for 895b7c16371d966da7eb221de358652c 2023-09-06T20:33:59,078 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/test-16940322964962/895b7c16371d966da7eb221de358652c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:33:59,078 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1093): Opened 895b7c16371d966da7eb221de358652c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10911037920, jitterRate=0.01616959273815155}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:33:59,079 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(992): Region open journal for 895b7c16371d966da7eb221de358652c: 2023-09-06T20:33:59,079 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:test-16940322964962,,1694032438327.895b7c16371d966da7eb221de358652c., pid=132, masterSystemTime=1694032439065 2023-09-06T20:33:59,081 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:test-16940322964962,,1694032438327.895b7c16371d966da7eb221de358652c. 2023-09-06T20:33:59,081 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] handler.AssignRegionHandler(158): Opened ns3:test-16940322964962,,1694032438327.895b7c16371d966da7eb221de358652c. 2023-09-06T20:33:59,081 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=131 updating hbase:meta row=895b7c16371d966da7eb221de358652c, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:33:59,084 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=132, resume processing ppid=131 2023-09-06T20:33:59,084 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=132, ppid=131, state=SUCCESS; OpenRegionProcedure 895b7c16371d966da7eb221de358652c, server=jenkins-hbase3.apache.org,36961,1694032287907 in 170 msec 2023-09-06T20:33:59,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=131, resume processing ppid=130 2023-09-06T20:33:59,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=131, ppid=130, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16940322964962, region=895b7c16371d966da7eb221de358652c, ASSIGN in 327 msec 2023-09-06T20:33:59,086 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16940322964962","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032439086"}]},"ts":"1694032439086"} 2023-09-06T20:33:59,087 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16940322964962, state=ENABLED in hbase:meta 2023-09-06T20:33:59,088 DEBUG [PEWorker-1 {}] procedure.TruncateTableProcedure(145): truncate 'ns3:test-16940322964962' completed 2023-09-06T20:33:59,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=130, state=SUCCESS; TruncateTableProcedure (table=ns3:test-16940322964962 preserveSplits=true) in 773 msec 2023-09-06T20:33:59,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=130 2023-09-06T20:33:59,425 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-09-06T20:33:59,425 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(233): Backup: backup_1694032419452 hdfs://localhost:39327/backupUT/backup_1694032419452/ns3/test-16940322964962/ 2023-09-06T20:33:59,425 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-09-06T20:33:59,425 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:33:59,446 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultipleOverwriteCommand Thread=605 (was 601) - Thread LEAK? -, OpenFileDescriptor=683 (was 675) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=151 (was 118) - SystemLoadAverage LEAK? -, ProcessCount=167 (was 167), AvailableMemoryMB=3532 (was 3042) - AvailableMemoryMB LEAK? - 2023-09-06T20:33:59,446 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(130): Thread=605 is superior to 500 2023-09-06T20:33:59,466 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultipleCommand Thread=605, OpenFileDescriptor=683, MaxFileDescriptor=60000, SystemLoadAverage=151, ProcessCount=167, AvailableMemoryMB=3532 2023-09-06T20:33:59,466 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(130): Thread=605 is superior to 500 2023-09-06T20:33:59,466 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] backup.TestFullRestore(144): create full backup image on multiple tables: command-line 2023-09-06T20:33:59,475 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dc2f96d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:59,478 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:59,480 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:34152, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:33:59,482 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:33:59,484 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:58152, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:33:59,493 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:33:59,498 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:33:59,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032439498 (current time:1694032439498). 2023-09-06T20:33:59,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:33:59,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:33:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34c058f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:59,506 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:34158, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:33:59,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:33:59,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:33:59,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=133, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:33:59,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 133 2023-09-06T20:33:59,512 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:33:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=133 2023-09-06T20:33:59,512 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:33:59,514 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:33:59,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=133 2023-09-06T20:33:59,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=133 2023-09-06T20:33:59,904 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:33:59,921 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:33:59,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281}] 2023-09-06T20:33:59,924 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=134, ppid=133, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 2023-09-06T20:34:00,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=134 2023-09-06T20:34:00,076 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:34:00,076 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HRegion(2819): Flushing cf63a85c53279bc94811b50578f31281 2/2 column families, dataSize=1.50 KB heapSize=2.73 KB 2023-09-06T20:34:00,081 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/3ff5f0a110934ea19f13a9dc5a7d2318 is 171, key is trslm:hdfs://localhost:39327/backupUT\x00ns2:test-16940322964961/meta:log-roll-map/1694032433354/Put/seqid=0 2023-09-06T20:34:00,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=133 2023-09-06T20:34:00,485 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=717 B at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/3ff5f0a110934ea19f13a9dc5a7d2318 2023-09-06T20:34:00,492 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/c1202b8538934be1bed1ea11b0457324 is 524, key is session:backup_1694032419452/session:context/1694032434189/Put/seqid=0 2023-09-06T20:34:00,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=133 2023-09-06T20:34:00,896 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=820 B at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/c1202b8538934be1bed1ea11b0457324 2023-09-06T20:34:00,902 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/3ff5f0a110934ea19f13a9dc5a7d2318 as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/3ff5f0a110934ea19f13a9dc5a7d2318 2023-09-06T20:34:00,906 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/3ff5f0a110934ea19f13a9dc5a7d2318, entries=6, sequenceid=105, filesize=5.8 K 2023-09-06T20:34:00,907 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/c1202b8538934be1bed1ea11b0457324 as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/c1202b8538934be1bed1ea11b0457324 2023-09-06T20:34:00,912 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/c1202b8538934be1bed1ea11b0457324, entries=2, sequenceid=105, filesize=5.5 K 2023-09-06T20:34:00,913 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HRegion(3022): Finished flush of dataSize ~1.50 KB/1537, heapSize ~2.70 KB/2760, currentSize=0 B/0 for cf63a85c53279bc94811b50578f31281 in 836ms, sequenceid=105, compaction requested=true 2023-09-06T20:34:00,913 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HRegion(2520): Flush status journal for cf63a85c53279bc94811b50578f31281: 2023-09-06T20:34:00,913 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. for snapshot_backup_system completed. 2023-09-06T20:34:00,913 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:34:00,913 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:34:00,913 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/0379b1e1f59e40d2ac2991b2ae6b86ce, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/dd7edbeade334fa7895ff079e0fc3472, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/78eec26cfa0242e48353b19a4a00b7d5, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/faffbcb72f8649bcad705936a3105fd4, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/01fb4a2eb68c4823807811190aa11f8b, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/b14bcdd382934779b0fa80ae042378aa, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/b7aca3aa47f24516bd2686bfd6b9193d, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/48f5b68b63f84de7a7c1f4f08993f64f, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/3ff5f0a110934ea19f13a9dc5a7d2318] hfiles 2023-09-06T20:34:00,913 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (1/9): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/0379b1e1f59e40d2ac2991b2ae6b86ce for snapshot=snapshot_backup_system 2023-09-06T20:34:00,913 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (2/9): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/dd7edbeade334fa7895ff079e0fc3472 for snapshot=snapshot_backup_system 2023-09-06T20:34:00,913 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (3/9): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/78eec26cfa0242e48353b19a4a00b7d5 for snapshot=snapshot_backup_system 2023-09-06T20:34:00,913 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (4/9): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/faffbcb72f8649bcad705936a3105fd4 for snapshot=snapshot_backup_system 2023-09-06T20:34:00,913 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (5/9): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/01fb4a2eb68c4823807811190aa11f8b for snapshot=snapshot_backup_system 2023-09-06T20:34:00,913 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (6/9): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/b14bcdd382934779b0fa80ae042378aa for snapshot=snapshot_backup_system 2023-09-06T20:34:00,913 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (7/9): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/b7aca3aa47f24516bd2686bfd6b9193d for snapshot=snapshot_backup_system 2023-09-06T20:34:00,913 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (8/9): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/48f5b68b63f84de7a7c1f4f08993f64f for snapshot=snapshot_backup_system 2023-09-06T20:34:00,913 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (9/9): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/3ff5f0a110934ea19f13a9dc5a7d2318 for snapshot=snapshot_backup_system 2023-09-06T20:34:00,913 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/4ced064cdbd34bd6b594c76f049fa4ee, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/baca9a30045a4dabad9b3b78a8fd6c86, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/bcfde09dc46641bebf14d756f321d0af, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/2dc68dbd9f3749a1a2d9d7333fa4771d, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/6da826e7b47e41daa41e268fb4e951b2, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/849e36507ab34f85a508bffbc6e215d3, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/8bd33efd4a8e44d1b5b9f6a66a166990, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/13357027918440549abec030e3035318, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/a2912410595b4f2d8ed499b1de1206c9, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/c1202b8538934be1bed1ea11b0457324] hfiles 2023-09-06T20:34:00,913 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (1/10): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/4ced064cdbd34bd6b594c76f049fa4ee for snapshot=snapshot_backup_system 2023-09-06T20:34:00,913 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (2/10): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/baca9a30045a4dabad9b3b78a8fd6c86 for snapshot=snapshot_backup_system 2023-09-06T20:34:00,913 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (3/10): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/bcfde09dc46641bebf14d756f321d0af for snapshot=snapshot_backup_system 2023-09-06T20:34:00,913 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (4/10): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/2dc68dbd9f3749a1a2d9d7333fa4771d for snapshot=snapshot_backup_system 2023-09-06T20:34:00,913 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (5/10): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/6da826e7b47e41daa41e268fb4e951b2 for snapshot=snapshot_backup_system 2023-09-06T20:34:00,913 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (6/10): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/849e36507ab34f85a508bffbc6e215d3 for snapshot=snapshot_backup_system 2023-09-06T20:34:00,913 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (7/10): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/8bd33efd4a8e44d1b5b9f6a66a166990 for snapshot=snapshot_backup_system 2023-09-06T20:34:00,913 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (8/10): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/13357027918440549abec030e3035318 for snapshot=snapshot_backup_system 2023-09-06T20:34:00,913 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (9/10): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/a2912410595b4f2d8ed499b1de1206c9 for snapshot=snapshot_backup_system 2023-09-06T20:34:00,913 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (10/10): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/c1202b8538934be1bed1ea11b0457324 for snapshot=snapshot_backup_system 2023-09-06T20:34:01,323 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:34:01,324 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=134 2023-09-06T20:34:01,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=134 2023-09-06T20:34:01,324 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region cf63a85c53279bc94811b50578f31281 2023-09-06T20:34:01,326 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=134, ppid=133, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 2023-09-06T20:34:01,328 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=134, resume processing ppid=133 2023-09-06T20:34:01,328 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:34:01,328 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=134, ppid=133, state=SUCCESS; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 in 1.4060 sec 2023-09-06T20:34:01,329 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:34:01,329 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:34:01,329 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:34:01,331 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:34:01,352 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:34:01,369 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:34:01,369 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:34:01,370 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:34:01,370 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 133 2023-09-06T20:34:01,372 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=133, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8600 sec 2023-09-06T20:34:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=133 2023-09-06T20:34:01,618 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(115): Backup backup_1694032439475 started at 1694032441618. 2023-09-06T20:34:01,620 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(119): Backup session backup_1694032439475 has been started. 2023-09-06T20:34:01,626 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-09-06T20:34:01,627 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-09-06T20:34:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-09-06T20:34:01,627 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-09-06T20:34:01,627 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-09-06T20:34:01,628 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-09-06T20:34:01,628 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-09-06T20:34:01,629 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:34:01,630 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:01,630 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:34:01,630 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:34:01,631 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-09-06T20:34:01,631 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-09-06T20:34:01,631 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-09-06T20:34:01,631 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:34:01,631 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-09-06T20:34:01,632 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-09-06T20:34:01,632 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-09-06T20:34:01,632 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-09-06T20:34:01,632 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-09-06T20:34:01,635 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-09-06T20:34:01,635 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-09-06T20:34:01,635 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,36961,1694032287907' joining acquired barrier for procedure (rolllog) in zk 2023-09-06T20:34:01,640 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-09-06T20:34:01,640 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:01,640 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:34:01,641 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:34:01,641 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-09-06T20:34:01,641 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:34:01,641 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:34:01,642 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:01,642 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:01,642 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:34:01,642 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,36961,1694032287907' joining acquired barrier for procedure 'rolllog' on coordinator 2023-09-06T20:34:01,643 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@13936cfe[Count = 0] remaining members to acquire global barrier 2023-09-06T20:34:01,643 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-09-06T20:34:01,643 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-09-06T20:34:01,644 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-09-06T20:34:01,644 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-09-06T20:34:01,644 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-09-06T20:34:01,644 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-09-06T20:34:01,644 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-09-06T20:34:01,644 DEBUG [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:01,644 INFO [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1694032423630 highest: 1694032423630 on jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:01,644 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta:.meta(num 1694032423612) roll requested 2023-09-06T20:34:01,656 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032441644.meta, exclude list is [], retry=0 2023-09-06T20:34:01,660 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK] 2023-09-06T20:34:01,661 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032423612.meta with entries=24, filesize=6.28 KB; new WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032441644.meta 2023-09-06T20:34:01,661 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK]] 2023-09-06T20:34:01,661 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032423612.meta is not closed yet, will try archiving it next time 2023-09-06T20:34:01,661 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36961%2C1694032287907:(num 1694032423630) roll requested 2023-09-06T20:34:01,674 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032441662, exclude list is [], retry=0 2023-09-06T20:34:01,677 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK] 2023-09-06T20:34:01,678 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032423630 with entries=17, filesize=5.81 KB; new WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032441662 2023-09-06T20:34:01,678 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK]] 2023-09-06T20:34:01,679 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032423630 is not closed yet, will try archiving it next time 2023-09-06T20:34:01,679 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032401249 to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/oldWALs/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032401249 2023-09-06T20:34:01,684 DEBUG [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-09-06T20:34:01,684 INFO [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1694032441662 on jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:01,692 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-09-06T20:34:01,692 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,36961,1694032287907' in zk 2023-09-06T20:34:01,695 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-09-06T20:34:01,695 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:34:01,695 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-09-06T20:34:01,696 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:01,696 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:34:01,696 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:34:01,696 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:34:01,697 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:34:01,697 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:01,697 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:01,698 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:34:01,698 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:01,699 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:01,699 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,36961,1694032287907': 2023-09-06T20:34:01,699 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,36961,1694032287907' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-09-06T20:34:01,699 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-09-06T20:34:01,700 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-09-06T20:34:01,700 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-09-06T20:34:01,700 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-09-06T20:34:01,700 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-09-06T20:34:01,701 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-09-06T20:34:01,701 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:34:01,701 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:34:01,701 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-09-06T20:34:01,701 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:34:01,702 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:34:01,702 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:34:01,702 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:34:01,702 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:01,702 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:34:01,702 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:34:01,703 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:01,703 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:01,703 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:34:01,703 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:01,704 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:01,709 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:34:01,709 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:34:01,709 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:34:01,709 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-09-06T20:34:01,709 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-09-06T20:34:01,710 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:34:01,710 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:34:01,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-09-06T20:34:01,814 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1694032441813_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } 2023-09-06T20:34:01,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032441814 (current time:1694032441814). 2023-09-06T20:34:01,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:34:01,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1694032441813_ns2_test-16940322964961 VERSION not specified, setting to 2 2023-09-06T20:34:01,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bf29cd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:01,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:01,823 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50280, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:01,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:01,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:34:01,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=135, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1694032441813_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } 2023-09-06T20:34:01,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1694032441813_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 }, snapshot procedure id = 135 2023-09-06T20:34:01,826 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1694032441813_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:34:01,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=135 2023-09-06T20:34:01,827 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1694032441813_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:34:01,829 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1694032441813_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:34:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=135 2023-09-06T20:34:02,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=135 2023-09-06T20:34:02,237 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1694032441813_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:34:02,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE; SnapshotRegionProcedure e70b91c8ad9e774e6ad3f358efadcaf1}] 2023-09-06T20:34:02,239 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=136, ppid=135, state=RUNNABLE; SnapshotRegionProcedure e70b91c8ad9e774e6ad3f358efadcaf1 2023-09-06T20:34:02,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=136 2023-09-06T20:34:02,391 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16940322964961,,1694032434553.e70b91c8ad9e774e6ad3f358efadcaf1. 2023-09-06T20:34:02,391 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] regionserver.HRegion(2520): Flush status journal for e70b91c8ad9e774e6ad3f358efadcaf1: 2023-09-06T20:34:02,391 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16940322964961,,1694032434553.e70b91c8ad9e774e6ad3f358efadcaf1. for snapshot_1694032441813_ns2_test-16940322964961 completed. 2023-09-06T20:34:02,391 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16940322964961,,1694032434553.e70b91c8ad9e774e6ad3f358efadcaf1.' region-info for snapshot=snapshot_1694032441813_ns2_test-16940322964961 2023-09-06T20:34:02,391 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:34:02,391 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/e70b91c8ad9e774e6ad3f358efadcaf1/f/033098d0c12f4e7492b697713877c54c_SeqId_4_] hfiles 2023-09-06T20:34:02,391 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/e70b91c8ad9e774e6ad3f358efadcaf1/f/033098d0c12f4e7492b697713877c54c_SeqId_4_ for snapshot=snapshot_1694032441813_ns2_test-16940322964961 2023-09-06T20:34:02,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=135 2023-09-06T20:34:02,797 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16940322964961,,1694032434553.e70b91c8ad9e774e6ad3f358efadcaf1. 2023-09-06T20:34:02,797 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=136 2023-09-06T20:34:02,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=136 2023-09-06T20:34:02,797 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1694032441813_ns2_test-16940322964961 on region e70b91c8ad9e774e6ad3f358efadcaf1 2023-09-06T20:34:02,799 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=136, ppid=135, state=RUNNABLE; SnapshotRegionProcedure e70b91c8ad9e774e6ad3f358efadcaf1 2023-09-06T20:34:02,801 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=136, resume processing ppid=135 2023-09-06T20:34:02,801 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1694032441813_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:34:02,801 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=136, ppid=135, state=SUCCESS; SnapshotRegionProcedure e70b91c8ad9e774e6ad3f358efadcaf1 in 563 msec 2023-09-06T20:34:02,802 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1694032441813_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:34:02,802 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1694032441813_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:34:02,802 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1694032441813_ns2_test-16940322964961 2023-09-06T20:34:02,803 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032441813_ns2_test-16940322964961 2023-09-06T20:34:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=135 2023-09-06T20:34:03,212 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1694032441813_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:34:03,217 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1694032441813_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:34:03,218 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032441813_ns2_test-16940322964961 to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032441813_ns2_test-16940322964961 2023-09-06T20:34:03,219 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1694032441813_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:34:03,219 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1694032441813_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 }, snapshot procedure id = 135 2023-09-06T20:34:03,220 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=135, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1694032441813_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } in 1.3950 sec 2023-09-06T20:34:03,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=135 2023-09-06T20:34:03,932 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1694032443932_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } 2023-09-06T20:34:03,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032443933 (current time:1694032443933). 2023-09-06T20:34:03,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:34:03,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1694032443932_ns3_test-16940322964962 VERSION not specified, setting to 2 2023-09-06T20:34:03,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a19ea4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:03,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:03,941 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50292, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:03,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:03,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:34:03,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=137, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1694032443932_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } 2023-09-06T20:34:03,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1694032443932_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 }, snapshot procedure id = 137 2023-09-06T20:34:03,944 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1694032443932_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:34:03,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=137 2023-09-06T20:34:03,945 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1694032443932_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:34:03,946 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1694032443932_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:34:04,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=137 2023-09-06T20:34:04,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=137 2023-09-06T20:34:04,352 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1694032443932_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:34:04,352 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE; SnapshotRegionProcedure 895b7c16371d966da7eb221de358652c}] 2023-09-06T20:34:04,353 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=138, ppid=137, state=RUNNABLE; SnapshotRegionProcedure 895b7c16371d966da7eb221de358652c 2023-09-06T20:34:04,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=138 2023-09-06T20:34:04,505 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16940322964962,,1694032438327.895b7c16371d966da7eb221de358652c. 2023-09-06T20:34:04,506 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] regionserver.HRegion(2520): Flush status journal for 895b7c16371d966da7eb221de358652c: 2023-09-06T20:34:04,506 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16940322964962,,1694032438327.895b7c16371d966da7eb221de358652c. for snapshot_1694032443932_ns3_test-16940322964962 completed. 2023-09-06T20:34:04,507 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16940322964962,,1694032438327.895b7c16371d966da7eb221de358652c.' region-info for snapshot=snapshot_1694032443932_ns3_test-16940322964962 2023-09-06T20:34:04,507 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:34:04,507 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-09-06T20:34:04,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=137 2023-09-06T20:34:04,912 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16940322964962,,1694032438327.895b7c16371d966da7eb221de358652c. 2023-09-06T20:34:04,912 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=138 2023-09-06T20:34:04,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=138 2023-09-06T20:34:04,913 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1694032443932_ns3_test-16940322964962 on region 895b7c16371d966da7eb221de358652c 2023-09-06T20:34:04,914 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=138, ppid=137, state=RUNNABLE; SnapshotRegionProcedure 895b7c16371d966da7eb221de358652c 2023-09-06T20:34:04,917 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=138, resume processing ppid=137 2023-09-06T20:34:04,917 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=138, ppid=137, state=SUCCESS; SnapshotRegionProcedure 895b7c16371d966da7eb221de358652c in 563 msec 2023-09-06T20:34:04,917 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1694032443932_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:34:04,917 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1694032443932_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:34:04,918 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1694032443932_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:34:04,918 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1694032443932_ns3_test-16940322964962 2023-09-06T20:34:04,919 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032443932_ns3_test-16940322964962 2023-09-06T20:34:05,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=137 2023-09-06T20:34:05,333 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1694032443932_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:34:05,338 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1694032443932_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:34:05,338 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032443932_ns3_test-16940322964962 to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032443932_ns3_test-16940322964962 2023-09-06T20:34:05,339 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1694032443932_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:34:05,339 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1694032443932_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 }, snapshot procedure id = 137 2023-09-06T20:34:05,340 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=137, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1694032443932_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } in 1.3970 sec 2023-09-06T20:34:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=137 2023-09-06T20:34:06,049 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1694032439475 2023-09-06T20:34:06,049 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-09-06T20:34:06,049 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-09-06T20:34:06,049 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1694032439475_ns3:test-16940322964962 2023-09-06T20:34:06,049 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1694032443932_ns3_test-16940322964962 to hdfs://localhost:39327/backupUT/backup_1694032439475/ns3/test-16940322964962/ 2023-09-06T20:34:06,049 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-09-06T20:34:06,072 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:34:06,072 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37419,1694032284546_-163320312_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:39327/backupUT/backup_1694032439475/ns3/test-16940322964962, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:39327/backupUT/backup_1694032439475/ns3/test-16940322964962/.hbase-snapshot/.tmp/snapshot_1694032443932_ns3_test-16940322964962 2023-09-06T20:34:06,072 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1, snapshotDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032443932_ns3_test-16940322964962. 2023-09-06T20:34:06,077 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032443932_ns3_test-16940322964962 to hdfs://localhost:39327/backupUT/backup_1694032439475/ns3/test-16940322964962/.hbase-snapshot/.tmp/snapshot_1694032443932_ns3_test-16940322964962 2023-09-06T20:34:06,490 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:06,490 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:06,491 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:06,491 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:07,456 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop-3743889072594930969.jar 2023-09-06T20:34:07,456 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:07,456 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:07,457 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:07,457 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:07,457 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:07,458 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:07,458 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:07,458 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:07,459 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:07,459 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:34:07,460 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:34:07,460 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:34:07,461 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:34:07,461 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:34:07,462 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:34:07,462 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:34:07,462 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:34:07,463 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:34:07,463 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:34:07,463 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:34:07,464 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:34:07,464 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:07,465 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:07,465 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:34:07,466 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:07,466 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:07,466 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:34:07,467 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:34:07,509 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1694032443932_ns3_test-16940322964962' hfile list 2023-09-06T20:34:07,581 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032447545/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:34:07,581 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032447545/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:34:08,611 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-09-06T20:34:08,612 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-09-06T20:34:08,617 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1694032443932_ns3_test-16940322964962 2023-09-06T20:34:08,617 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1694032443932_ns3_test-16940322964962 finished. 2023-09-06T20:34:08,617 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1694032439475_ns2:test-16940322964961 2023-09-06T20:34:08,617 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1694032441813_ns2_test-16940322964961 to hdfs://localhost:39327/backupUT/backup_1694032439475/ns2/test-16940322964961/ 2023-09-06T20:34:08,617 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-09-06T20:34:08,639 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:34:08,639 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37419,1694032284546_232406832_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:39327/backupUT/backup_1694032439475/ns2/test-16940322964961, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:39327/backupUT/backup_1694032439475/ns2/test-16940322964961/.hbase-snapshot/.tmp/snapshot_1694032441813_ns2_test-16940322964961 2023-09-06T20:34:08,639 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1, snapshotDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032441813_ns2_test-16940322964961. 2023-09-06T20:34:08,646 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032441813_ns2_test-16940322964961 to hdfs://localhost:39327/backupUT/backup_1694032439475/ns2/test-16940322964961/.hbase-snapshot/.tmp/snapshot_1694032441813_ns2_test-16940322964961 2023-09-06T20:34:09,058 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:09,058 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:09,059 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:09,059 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:10,058 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop-5991403692888520901.jar 2023-09-06T20:34:10,058 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:10,058 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:10,059 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:10,059 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:10,059 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:10,060 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:10,060 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:10,060 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:10,061 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:10,061 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:34:10,062 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:34:10,062 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:34:10,063 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:34:10,063 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:34:10,063 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:34:10,064 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:34:10,064 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:34:10,064 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:34:10,065 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:34:10,065 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:34:10,065 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:34:10,066 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:10,066 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:10,067 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:34:10,067 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:10,067 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:10,068 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:34:10,069 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:34:10,112 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1694032441813_ns2_test-16940322964961' hfile list 2023-09-06T20:34:10,114 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-09-06T20:34:10,180 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032450148/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:34:10,180 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032450148/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:34:10,266 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-09-06T20:34:10,292 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/e70b91c8ad9e774e6ad3f358efadcaf1/f/033098d0c12f4e7492b697713877c54c_SeqId_4_ output=hdfs://localhost:39327/backupUT/backup_1694032439475/ns2/test-16940322964961/archive/data/ns2/test-16940322964961/e70b91c8ad9e774e6ad3f358efadcaf1/f/033098d0c12f4e7492b697713877c54c_SeqId_4_ 2023-09-06T20:34:10,292 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8725 (8.5 K) time=0sec 8.321M/sec 2023-09-06T20:34:11,245 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-09-06T20:34:11,246 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-09-06T20:34:11,251 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1694032441813_ns2_test-16940322964961 2023-09-06T20:34:11,251 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1694032441813_ns2_test-16940322964961 finished. 2023-09-06T20:34:11,251 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1014): ns3:test-16940322964962 2023-09-06T20:34:11,251 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1014): ns2:test-16940322964961 2023-09-06T20:34:11,257 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1694032439475 2023-09-06T20:34:11,257 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-09-06T20:34:11,663 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:39327/backupUT/backup_1694032439475/.backup.manifest 2023-09-06T20:34:11,663 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1694032439475 2023-09-06T20:34:11,663 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-09-06T20:34:12,069 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:39327/backupUT/backup_1694032439475/.backup.manifest 2023-09-06T20:34:12,069 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(358): Backup backup_1694032439475 finished: type=FULL,tablelist=ns3:test-16940322964962;ns2:test-16940322964961,targetRootDir=hdfs://localhost:39327/backupUT,startts=1694032441618,completets=1694032451257,bytescopied=0 2023-09-06T20:34:12,069 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-09-06T20:34:12,069 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1694032443932_ns3_test-16940322964962 2023-09-06T20:34:12,070 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1694032443932_ns3_test-16940322964962" 2023-09-06T20:34:12,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1694032443932_ns3_test-16940322964962 2023-09-06T20:34:12,073 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1694032443932_ns3_test-16940322964962 for backup backup_1694032439475 succeeded. 2023-09-06T20:34:12,073 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1694032441813_ns2_test-16940322964961 2023-09-06T20:34:12,074 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1694032441813_ns2_test-16940322964961" 2023-09-06T20:34:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1694032441813_ns2_test-16940322964961 2023-09-06T20:34:12,076 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1694032441813_ns2_test-16940322964961 for backup backup_1694032439475 succeeded. 2023-09-06T20:34:12,076 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:34:12,080 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:34:12,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:34:12,082 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:34:12,083 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:34:12,088 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(376): Backup backup_1694032439475 completed. 2023-09-06T20:34:12,089 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:12,107 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5db472da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:12,108 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:12,110 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45912, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:12,110 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(101): HBase table ns2:table2 does not exist. It will be created during restore process 2023-09-06T20:34:12,111 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(101): HBase table ns3:table3_restore does not exist. It will be created during restore process 2023-09-06T20:34:12,111 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:39327/backupUT/backup_1694032439475/.backup.manifest 2023-09-06T20:34:12,113 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032439475/.backup.manifest 2023-09-06T20:34:12,114 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:39327/backupUT/backup_1694032439475/.backup.manifest 2023-09-06T20:34:12,115 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032439475/.backup.manifest 2023-09-06T20:34:12,115 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:39327/backupUT/backup_1694032439475/.backup.manifest 2023-09-06T20:34:12,117 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032439475/.backup.manifest 2023-09-06T20:34:12,117 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(148): Restoring 'ns2:test-16940322964961' to 'ns2:table2' from full backup image hdfs://localhost:39327/backupUT/backup_1694032439475/ns2/test-16940322964961 2023-09-06T20:34:12,123 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(487): Creating target table 'ns2:table2' 2023-09-06T20:34:12,123 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:39327/backupUT/backup_1694032439475/ns2/test-16940322964961/archive/data/ns2/test-16940322964961/e70b91c8ad9e774e6ad3f358efadcaf1 2023-09-06T20:34:12,123 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:39327/backupUT/backup_1694032439475/ns2/test-16940322964961/archive/data/ns2/test-16940322964961/e70b91c8ad9e774e6ad3f358efadcaf1/f in region [hdfs://localhost:39327/backupUT/backup_1694032439475/ns2/test-16940322964961/archive/data/ns2/test-16940322964961/e70b91c8ad9e774e6ad3f358efadcaf1] 2023-09-06T20:34:12,126 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:39327/backupUT/backup_1694032439475/ns2/test-16940322964961/archive/data/ns2/test-16940322964961/e70b91c8ad9e774e6ad3f358efadcaf1/f/033098d0c12f4e7492b697713877c54c_SeqId_4_ first=row0 last=row98 2023-09-06T20:34:12,126 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:34:12,128 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:33042, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:34:12,129 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:34:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=139, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:table2 2023-09-06T20:34:12,131 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:34:12,132 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "table2" procId is: 139 2023-09-06T20:34:12,132 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:34:12,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=139 2023-09-06T20:34:12,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=139 2023-09-06T20:34:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=139 2023-09-06T20:34:12,539 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 66249e4e8d7eb101947abdf29a8225a3, NAME => 'ns2:table2,,1694032452129.66249e4e8d7eb101947abdf29a8225a3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:34:12,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=139 2023-09-06T20:34:12,945 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:table2,,1694032452129.66249e4e8d7eb101947abdf29a8225a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:34:12,945 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1668): Closing 66249e4e8d7eb101947abdf29a8225a3, disabling compactions & flushes 2023-09-06T20:34:12,945 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:table2,,1694032452129.66249e4e8d7eb101947abdf29a8225a3. 2023-09-06T20:34:12,945 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:table2,,1694032452129.66249e4e8d7eb101947abdf29a8225a3. 2023-09-06T20:34:12,945 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:table2,,1694032452129.66249e4e8d7eb101947abdf29a8225a3. after waiting 0 ms 2023-09-06T20:34:12,945 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:table2,,1694032452129.66249e4e8d7eb101947abdf29a8225a3. 2023-09-06T20:34:12,945 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1909): Closed ns2:table2,,1694032452129.66249e4e8d7eb101947abdf29a8225a3. 2023-09-06T20:34:12,945 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1622): Region close journal for 66249e4e8d7eb101947abdf29a8225a3: 2023-09-06T20:34:12,946 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:34:12,946 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:table2,,1694032452129.66249e4e8d7eb101947abdf29a8225a3.","families":{"info":[{"qualifier":"regioninfo","vlen":36,"tag":[],"timestamp":"1694032452946"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032452946"}]},"ts":"1694032452946"} 2023-09-06T20:34:12,948 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:34:12,948 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:34:12,948 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032452948"}]},"ts":"1694032452948"} 2023-09-06T20:34:12,949 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=ENABLING in hbase:meta 2023-09-06T20:34:12,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:table2, region=66249e4e8d7eb101947abdf29a8225a3, ASSIGN}] 2023-09-06T20:34:12,957 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=140, ppid=139, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:table2, region=66249e4e8d7eb101947abdf29a8225a3, ASSIGN 2023-09-06T20:34:12,958 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=140, ppid=139, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:table2, region=66249e4e8d7eb101947abdf29a8225a3, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36961,1694032287907; forceNewPlan=false, retain=false 2023-09-06T20:34:13,109 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=140 updating hbase:meta row=66249e4e8d7eb101947abdf29a8225a3, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:13,111 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE; OpenRegionProcedure 66249e4e8d7eb101947abdf29a8225a3, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:34:13,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=139 2023-09-06T20:34:13,265 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(130): Open ns2:table2,,1694032452129.66249e4e8d7eb101947abdf29a8225a3. 2023-09-06T20:34:13,265 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7254): Opening region: {ENCODED => 66249e4e8d7eb101947abdf29a8225a3, NAME => 'ns2:table2,,1694032452129.66249e4e8d7eb101947abdf29a8225a3.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:34:13,266 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:34:13,266 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table2 66249e4e8d7eb101947abdf29a8225a3 2023-09-06T20:34:13,266 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(888): Instantiated ns2:table2,,1694032452129.66249e4e8d7eb101947abdf29a8225a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:34:13,266 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7296): checking encryption for 66249e4e8d7eb101947abdf29a8225a3 2023-09-06T20:34:13,266 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7299): checking classloading for 66249e4e8d7eb101947abdf29a8225a3 2023-09-06T20:34:13,268 INFO [StoreOpener-66249e4e8d7eb101947abdf29a8225a3-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 66249e4e8d7eb101947abdf29a8225a3 2023-09-06T20:34:13,269 INFO [StoreOpener-66249e4e8d7eb101947abdf29a8225a3-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66249e4e8d7eb101947abdf29a8225a3 columnFamilyName f 2023-09-06T20:34:13,269 DEBUG [StoreOpener-66249e4e8d7eb101947abdf29a8225a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:34:13,270 INFO [StoreOpener-66249e4e8d7eb101947abdf29a8225a3-1 {}] regionserver.HStore(324): Store=66249e4e8d7eb101947abdf29a8225a3/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:34:13,270 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/table2/66249e4e8d7eb101947abdf29a8225a3 2023-09-06T20:34:13,271 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/table2/66249e4e8d7eb101947abdf29a8225a3 2023-09-06T20:34:13,272 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1076): writing seq id for 66249e4e8d7eb101947abdf29a8225a3 2023-09-06T20:34:13,274 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/table2/66249e4e8d7eb101947abdf29a8225a3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:34:13,274 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1093): Opened 66249e4e8d7eb101947abdf29a8225a3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10518273440, jitterRate=-0.02040944993495941}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:34:13,275 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(992): Region open journal for 66249e4e8d7eb101947abdf29a8225a3: 2023-09-06T20:34:13,275 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:table2,,1694032452129.66249e4e8d7eb101947abdf29a8225a3., pid=141, masterSystemTime=1694032453262 2023-09-06T20:34:13,277 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:table2,,1694032452129.66249e4e8d7eb101947abdf29a8225a3. 2023-09-06T20:34:13,277 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(158): Opened ns2:table2,,1694032452129.66249e4e8d7eb101947abdf29a8225a3. 2023-09-06T20:34:13,277 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=140 updating hbase:meta row=66249e4e8d7eb101947abdf29a8225a3, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:13,279 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=141, resume processing ppid=140 2023-09-06T20:34:13,279 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=141, ppid=140, state=SUCCESS; OpenRegionProcedure 66249e4e8d7eb101947abdf29a8225a3, server=jenkins-hbase3.apache.org,36961,1694032287907 in 167 msec 2023-09-06T20:34:13,280 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=140, resume processing ppid=139 2023-09-06T20:34:13,280 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=140, ppid=139, state=SUCCESS; TransitRegionStateProcedure table=ns2:table2, region=66249e4e8d7eb101947abdf29a8225a3, ASSIGN in 323 msec 2023-09-06T20:34:13,281 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:34:13,281 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032453281"}]},"ts":"1694032453281"} 2023-09-06T20:34:13,282 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=ENABLED in hbase:meta 2023-09-06T20:34:13,286 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:34:13,287 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=139, state=SUCCESS; CreateTableProcedure table=ns2:table2 in 1.1570 sec 2023-09-06T20:34:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=139 2023-09-06T20:34:14,244 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:39327/backupUT/backup_1694032439475/ns2/test-16940322964961/archive/data/ns2/test-16940322964961/e70b91c8ad9e774e6ad3f358efadcaf1 from hbase tables ns2:test-16940322964961 to tables ns2:table2 2023-09-06T20:34:14,244 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-16940322964961 into ns2:table2 2023-09-06T20:34:14,246 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-table2-1694032454244 from hdfs://localhost:39327/backupUT/backup_1694032439475/ns2/test-16940322964961/archive/data/ns2/test-16940322964961/e70b91c8ad9e774e6ad3f358efadcaf1 2023-09-06T20:34:14,251 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5374742c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:14,251 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:34:14,252 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:33058, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:34:14,253 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-09-06T20:34:14,253 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:table2 2023-09-06T20:34:14,258 DEBUG [hconnection-0x1693a8f7-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:14,258 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45914, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:14,260 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:table2: 2023-09-06T20:34:14,260 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-09-06T20:34:14,260 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/partitions_978ff6e1-f394-48d9-b4b2-a6b34e27fcb0 2023-09-06T20:34:14,667 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:14,667 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:14,668 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:14,669 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:15,643 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop-739053087157714210.jar 2023-09-06T20:34:15,643 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:15,644 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:15,644 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:15,644 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:15,645 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:15,645 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:15,645 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:15,646 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:15,646 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:15,647 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:34:15,647 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:34:15,648 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:34:15,648 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:34:15,648 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:34:15,649 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:34:15,649 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:34:15,649 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:34:15,650 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:34:15,650 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:34:15,651 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:34:15,651 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:34:15,651 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:15,652 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:15,652 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:15,652 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:15,652 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:15,653 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:15,653 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:34:15,654 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:table2 2023-09-06T20:34:15,654 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:15,654 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-09-06T20:34:15,654 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-09-06T20:34:15,655 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:34:15,666 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-09-06T20:34:15,745 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032455711/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:34:15,745 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032455711/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:34:15,747 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_table2 2023-09-06T20:34:15,747 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_table2 Metrics about Tables on a single HBase RegionServer 2023-09-06T20:34:15,793 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:39327/backupUT/backup_1694032439475/ns2/test-16940322964961/archive/data/ns2/test-16940322964961/e70b91c8ad9e774e6ad3f358efadcaf1/f/033098d0c12f4e7492b697713877c54c_SeqId_4_ 2023-09-06T20:34:15,797 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-09-06T20:34:15,815 DEBUG [pool-484-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3de1760f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:15,816 DEBUG [hconnection-0x169453e5-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:15,818 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45926, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:15,821 DEBUG [pool-484-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:15,821 DEBUG [pool-484-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-09-06T20:34:15,821 DEBUG [pool-484-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-09-06T20:34:15,826 DEBUG [pool-484-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-table2-1694032454244/_temporary/0/_temporary/attempt_local397730705_0022_r_000000_0/f/2df99285182048c0a052912782c169b7 is 37, key is row10/f:q1/1694032300351/Put/seqid=0 2023-09-06T20:34:15,827 WARN [Thread-2114 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:36961] Specified favored nodes: [jenkins-hbase3.apache.org:36961] 2023-09-06T20:34:16,791 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-table2-1694032454244 2023-09-06T20:34:16,798 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26915b80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:16,799 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:16,801 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45938, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:16,806 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-table2-1694032454244/_SUCCESS 2023-09-06T20:34:16,809 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:34:16,811 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:33064, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:34:16,821 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-table2-1694032454244/f/2df99285182048c0a052912782c169b7 first=Optional[row0] last=Optional[row98] 2023-09-06T20:34:16,822 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=ns2:table2,,1694032452129.66249e4e8d7eb101947abdf29a8225a3., hostname=jenkins-hbase3.apache.org,36961,1694032287907, seqNum=2 for row with hfile group [{f,hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-table2-1694032454244/f/2df99285182048c0a052912782c169b7}] 2023-09-06T20:34:16,827 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-table2-1694032454244/f/2df99285182048c0a052912782c169b7 for inclusion in 66249e4e8d7eb101947abdf29a8225a3/f 2023-09-06T20:34:16,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-09-06T20:34:16,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(625): Region bounds: first= last= 2023-09-06T20:34:16,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HRegion(2520): Flush status journal for 66249e4e8d7eb101947abdf29a8225a3: 2023-09-06T20:34:16,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-ns2-table2-1694032454244/f/2df99285182048c0a052912782c169b7 to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__ns2_table2__8lv4o6hr49s76dob6tckk8tbbeo7odie6gfej45gph2usvksmtucg1gria12omm4/f/2df99285182048c0a052912782c169b7 2023-09-06T20:34:16,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__ns2_table2__8lv4o6hr49s76dob6tckk8tbbeo7odie6gfej45gph2usvksmtucg1gria12omm4/f/2df99285182048c0a052912782c169b7 as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/table2/66249e4e8d7eb101947abdf29a8225a3/f/88d14cfddf4945658775deca14ebd4f6_SeqId_4_ 2023-09-06T20:34:16,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56e5d3bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:16,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:34:16,840 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:33080, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:34:16,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:16,844 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45952, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:16,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:16,855 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__ns2_table2__8lv4o6hr49s76dob6tckk8tbbeo7odie6gfej45gph2usvksmtucg1gria12omm4/f/2df99285182048c0a052912782c169b7 into 66249e4e8d7eb101947abdf29a8225a3/f as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/table2/66249e4e8d7eb101947abdf29a8225a3/f/88d14cfddf4945658775deca14ebd4f6_SeqId_4_ - updating store file list. 2023-09-06T20:34:16,860 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/table2/66249e4e8d7eb101947abdf29a8225a3/f/88d14cfddf4945658775deca14ebd4f6_SeqId_4_ into 66249e4e8d7eb101947abdf29a8225a3/f 2023-09-06T20:34:16,860 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__ns2_table2__8lv4o6hr49s76dob6tckk8tbbeo7odie6gfej45gph2usvksmtucg1gria12omm4/f/2df99285182048c0a052912782c169b7 into 66249e4e8d7eb101947abdf29a8225a3/f (new location: hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/table2/66249e4e8d7eb101947abdf29a8225a3/f/88d14cfddf4945658775deca14ebd4f6_SeqId_4_) 2023-09-06T20:34:16,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__ns2_table2__8lv4o6hr49s76dob6tckk8tbbeo7odie6gfej45gph2usvksmtucg1gria12omm4/f/2df99285182048c0a052912782c169b7 2023-09-06T20:34:16,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@202aec46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:16,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:34:16,871 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:33092, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-09-06T20:34:16,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:16,876 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45954, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-09-06T20:34:16,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:16,888 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:16,888 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-09-06T20:34:16,888 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-09-06T20:34:16,888 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(233): Backup: backup_1694032439475 hdfs://localhost:39327/backupUT/backup_1694032439475/ns2/test-16940322964961/ 2023-09-06T20:34:16,889 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:39327/backupUT/backup_1694032439475/.backup.manifest 2023-09-06T20:34:16,892 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032439475/.backup.manifest 2023-09-06T20:34:16,892 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(148): Restoring 'ns3:test-16940322964962' to 'ns3:table3_restore' from full backup image hdfs://localhost:39327/backupUT/backup_1694032439475/ns3/test-16940322964962 2023-09-06T20:34:16,896 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(101): Folder tableArchivePath: hdfs://localhost:39327/backupUT/backup_1694032439475/ns3/test-16940322964962/archive/data/ns3/test-16940322964962 does not exists 2023-09-06T20:34:16,896 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(326): find table descriptor but no archive dir for table ns3:test-16940322964962, will only create table 2023-09-06T20:34:16,898 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(487): Creating target table 'ns3:table3_restore' 2023-09-06T20:34:16,899 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns3:table3_restore', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:34:16,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:table3_restore 2023-09-06T20:34:16,901 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:34:16,901 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "table3_restore" procId is: 142 2023-09-06T20:34:16,902 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:34:16,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=142 2023-09-06T20:34:17,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=142 2023-09-06T20:34:17,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=142 2023-09-06T20:34:17,308 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 5b2405ce5bd5d4fc185b170750bb146c, NAME => 'ns3:table3_restore,,1694032456899.5b2405ce5bd5d4fc185b170750bb146c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:table3_restore', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:34:17,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=142 2023-09-06T20:34:17,714 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:table3_restore,,1694032456899.5b2405ce5bd5d4fc185b170750bb146c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:34:17,714 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1668): Closing 5b2405ce5bd5d4fc185b170750bb146c, disabling compactions & flushes 2023-09-06T20:34:17,715 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:table3_restore,,1694032456899.5b2405ce5bd5d4fc185b170750bb146c. 2023-09-06T20:34:17,715 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:table3_restore,,1694032456899.5b2405ce5bd5d4fc185b170750bb146c. 2023-09-06T20:34:17,715 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:table3_restore,,1694032456899.5b2405ce5bd5d4fc185b170750bb146c. after waiting 0 ms 2023-09-06T20:34:17,715 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:table3_restore,,1694032456899.5b2405ce5bd5d4fc185b170750bb146c. 2023-09-06T20:34:17,715 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1909): Closed ns3:table3_restore,,1694032456899.5b2405ce5bd5d4fc185b170750bb146c. 2023-09-06T20:34:17,715 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1622): Region close journal for 5b2405ce5bd5d4fc185b170750bb146c: 2023-09-06T20:34:17,716 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:34:17,716 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:table3_restore,,1694032456899.5b2405ce5bd5d4fc185b170750bb146c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1694032457716"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032457716"}]},"ts":"1694032457716"} 2023-09-06T20:34:17,717 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:34:17,717 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:34:17,718 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032457718"}]},"ts":"1694032457718"} 2023-09-06T20:34:17,719 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=ENABLING in hbase:meta 2023-09-06T20:34:17,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:table3_restore, region=5b2405ce5bd5d4fc185b170750bb146c, ASSIGN}] 2023-09-06T20:34:17,724 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:table3_restore, region=5b2405ce5bd5d4fc185b170750bb146c, ASSIGN 2023-09-06T20:34:17,725 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:table3_restore, region=5b2405ce5bd5d4fc185b170750bb146c, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36961,1694032287907; forceNewPlan=false, retain=false 2023-09-06T20:34:17,876 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=143 updating hbase:meta row=5b2405ce5bd5d4fc185b170750bb146c, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:17,878 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE; OpenRegionProcedure 5b2405ce5bd5d4fc185b170750bb146c, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:34:18,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=142 2023-09-06T20:34:18,032 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(130): Open ns3:table3_restore,,1694032456899.5b2405ce5bd5d4fc185b170750bb146c. 2023-09-06T20:34:18,032 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7254): Opening region: {ENCODED => 5b2405ce5bd5d4fc185b170750bb146c, NAME => 'ns3:table3_restore,,1694032456899.5b2405ce5bd5d4fc185b170750bb146c.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:34:18,032 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:34:18,032 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table3_restore 5b2405ce5bd5d4fc185b170750bb146c 2023-09-06T20:34:18,032 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(888): Instantiated ns3:table3_restore,,1694032456899.5b2405ce5bd5d4fc185b170750bb146c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:34:18,033 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7296): checking encryption for 5b2405ce5bd5d4fc185b170750bb146c 2023-09-06T20:34:18,033 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7299): checking classloading for 5b2405ce5bd5d4fc185b170750bb146c 2023-09-06T20:34:18,034 INFO [StoreOpener-5b2405ce5bd5d4fc185b170750bb146c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 5b2405ce5bd5d4fc185b170750bb146c 2023-09-06T20:34:18,035 INFO [StoreOpener-5b2405ce5bd5d4fc185b170750bb146c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b2405ce5bd5d4fc185b170750bb146c columnFamilyName f 2023-09-06T20:34:18,035 DEBUG [StoreOpener-5b2405ce5bd5d4fc185b170750bb146c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:34:18,035 INFO [StoreOpener-5b2405ce5bd5d4fc185b170750bb146c-1 {}] regionserver.HStore(324): Store=5b2405ce5bd5d4fc185b170750bb146c/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:34:18,036 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/table3_restore/5b2405ce5bd5d4fc185b170750bb146c 2023-09-06T20:34:18,036 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/table3_restore/5b2405ce5bd5d4fc185b170750bb146c 2023-09-06T20:34:18,038 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1076): writing seq id for 5b2405ce5bd5d4fc185b170750bb146c 2023-09-06T20:34:18,040 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/table3_restore/5b2405ce5bd5d4fc185b170750bb146c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:34:18,040 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): Opened 5b2405ce5bd5d4fc185b170750bb146c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11436409760, jitterRate=0.06509865820407867}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:34:18,041 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(992): Region open journal for 5b2405ce5bd5d4fc185b170750bb146c: 2023-09-06T20:34:18,041 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:table3_restore,,1694032456899.5b2405ce5bd5d4fc185b170750bb146c., pid=144, masterSystemTime=1694032458029 2023-09-06T20:34:18,043 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:table3_restore,,1694032456899.5b2405ce5bd5d4fc185b170750bb146c. 2023-09-06T20:34:18,043 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(158): Opened ns3:table3_restore,,1694032456899.5b2405ce5bd5d4fc185b170750bb146c. 2023-09-06T20:34:18,043 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=143 updating hbase:meta row=5b2405ce5bd5d4fc185b170750bb146c, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:18,045 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=144, resume processing ppid=143 2023-09-06T20:34:18,045 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=144, ppid=143, state=SUCCESS; OpenRegionProcedure 5b2405ce5bd5d4fc185b170750bb146c, server=jenkins-hbase3.apache.org,36961,1694032287907 in 166 msec 2023-09-06T20:34:18,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=143, resume processing ppid=142 2023-09-06T20:34:18,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=143, ppid=142, state=SUCCESS; TransitRegionStateProcedure table=ns3:table3_restore, region=5b2405ce5bd5d4fc185b170750bb146c, ASSIGN in 323 msec 2023-09-06T20:34:18,047 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:34:18,047 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032458047"}]},"ts":"1694032458047"} 2023-09-06T20:34:18,048 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=ENABLED in hbase:meta 2023-09-06T20:34:18,050 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:34:18,051 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=142, state=SUCCESS; CreateTableProcedure table=ns3:table3_restore in 1.1510 sec 2023-09-06T20:34:18,254 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:table2' 2023-09-06T20:34:18,255 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:table3_restore' 2023-09-06T20:34:18,806 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:34:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=142 2023-09-06T20:34:19,010 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-09-06T20:34:19,010 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(233): Backup: backup_1694032439475 hdfs://localhost:39327/backupUT/backup_1694032439475/ns3/test-16940322964962/ 2023-09-06T20:34:19,010 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-09-06T20:34:19,010 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:19,012 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable ns2:table2 2023-09-06T20:34:19,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=145, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns2:table2 2023-09-06T20:34:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=145 2023-09-06T20:34:19,015 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032459015"}]},"ts":"1694032459015"} 2023-09-06T20:34:19,016 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=DISABLING in hbase:meta 2023-09-06T20:34:19,018 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(293): Set ns2:table2 to state=DISABLING 2023-09-06T20:34:19,018 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:table2, region=66249e4e8d7eb101947abdf29a8225a3, UNASSIGN}] 2023-09-06T20:34:19,020 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=146, ppid=145, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:table2, region=66249e4e8d7eb101947abdf29a8225a3, UNASSIGN 2023-09-06T20:34:19,020 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=146 updating hbase:meta row=66249e4e8d7eb101947abdf29a8225a3, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:19,021 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE; CloseRegionProcedure 66249e4e8d7eb101947abdf29a8225a3, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:34:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=145 2023-09-06T20:34:19,173 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] handler.UnassignRegionHandler(119): Close 66249e4e8d7eb101947abdf29a8225a3 2023-09-06T20:34:19,173 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1668): Closing 66249e4e8d7eb101947abdf29a8225a3, disabling compactions & flushes 2023-09-06T20:34:19,173 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1690): Closing region ns2:table2,,1694032452129.66249e4e8d7eb101947abdf29a8225a3. 2023-09-06T20:34:19,173 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:table2,,1694032452129.66249e4e8d7eb101947abdf29a8225a3. 2023-09-06T20:34:19,173 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1778): Acquired close lock on ns2:table2,,1694032452129.66249e4e8d7eb101947abdf29a8225a3. after waiting 0 ms 2023-09-06T20:34:19,173 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1788): Updates disabled for region ns2:table2,,1694032452129.66249e4e8d7eb101947abdf29a8225a3. 2023-09-06T20:34:19,179 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/table2/66249e4e8d7eb101947abdf29a8225a3/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-09-06T20:34:19,179 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:34:19,180 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1909): Closed ns2:table2,,1694032452129.66249e4e8d7eb101947abdf29a8225a3. 2023-09-06T20:34:19,180 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1622): Region close journal for 66249e4e8d7eb101947abdf29a8225a3: 2023-09-06T20:34:19,181 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] handler.UnassignRegionHandler(163): Closed 66249e4e8d7eb101947abdf29a8225a3 2023-09-06T20:34:19,182 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=146 updating hbase:meta row=66249e4e8d7eb101947abdf29a8225a3, regionState=CLOSED 2023-09-06T20:34:19,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=147, resume processing ppid=146 2023-09-06T20:34:19,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=147, ppid=146, state=SUCCESS; CloseRegionProcedure 66249e4e8d7eb101947abdf29a8225a3, server=jenkins-hbase3.apache.org,36961,1694032287907 in 162 msec 2023-09-06T20:34:19,185 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=146, resume processing ppid=145 2023-09-06T20:34:19,185 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=146, ppid=145, state=SUCCESS; TransitRegionStateProcedure table=ns2:table2, region=66249e4e8d7eb101947abdf29a8225a3, UNASSIGN in 166 msec 2023-09-06T20:34:19,186 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032459185"}]},"ts":"1694032459185"} 2023-09-06T20:34:19,186 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=DISABLED in hbase:meta 2023-09-06T20:34:19,188 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(305): Set ns2:table2 to state=DISABLED 2023-09-06T20:34:19,189 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=145, state=SUCCESS; DisableTableProcedure table=ns2:table2 in 176 msec 2023-09-06T20:34:19,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=145 2023-09-06T20:34:19,318 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$5(2448): Client=jenkins//172.31.12.81 delete ns2:table2 2023-09-06T20:34:19,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=148, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=ns2:table2 2023-09-06T20:34:19,321 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=148, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=ns2:table2 2023-09-06T20:34:19,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=148 2023-09-06T20:34:19,321 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=148, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=ns2:table2 2023-09-06T20:34:19,323 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/table2/66249e4e8d7eb101947abdf29a8225a3 2023-09-06T20:34:19,325 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/table2/66249e4e8d7eb101947abdf29a8225a3/f, FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/table2/66249e4e8d7eb101947abdf29a8225a3/recovered.edits] 2023-09-06T20:34:19,328 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/table2/66249e4e8d7eb101947abdf29a8225a3/f/88d14cfddf4945658775deca14ebd4f6_SeqId_4_ to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/archive/data/ns2/table2/66249e4e8d7eb101947abdf29a8225a3/f/88d14cfddf4945658775deca14ebd4f6_SeqId_4_ 2023-09-06T20:34:19,331 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/table2/66249e4e8d7eb101947abdf29a8225a3/recovered.edits/6.seqid to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/archive/data/ns2/table2/66249e4e8d7eb101947abdf29a8225a3/recovered.edits/6.seqid 2023-09-06T20:34:19,331 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/table2/66249e4e8d7eb101947abdf29a8225a3 2023-09-06T20:34:19,331 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(293): Archived ns2:table2 regions 2023-09-06T20:34:19,333 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=148, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=ns2:table2 2023-09-06T20:34:19,336 ERROR [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36961 {}] util.ResourceLeakDetector(319): LEAK: RefCnt.release() was not called before it's garbage-collected. See https://netty.io/wiki/reference-counted-objects.html for more information. Recent access records: Created at: org.apache.hadoop.hbase.nio.RefCnt.(RefCnt.java:59) org.apache.hadoop.hbase.nio.RefCnt.create(RefCnt.java:54) org.apache.hadoop.hbase.regionserver.MemStoreLABImpl.(MemStoreLABImpl.java:108) sun.reflect.GeneratedConstructorAccessor92.newInstance(Unknown Source) sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) java.lang.reflect.Constructor.newInstance(Constructor.java:423) org.apache.hadoop.hbase.util.ReflectionUtils.instantiate(ReflectionUtils.java:55) org.apache.hadoop.hbase.util.ReflectionUtils.instantiateWithCustomCtor(ReflectionUtils.java:43) org.apache.hadoop.hbase.regionserver.MemStoreLAB.newInstance(MemStoreLAB.java:116) org.apache.hadoop.hbase.regionserver.SegmentFactory.createMutableSegment(SegmentFactory.java:81) org.apache.hadoop.hbase.regionserver.AbstractMemStore.resetActive(AbstractMemStore.java:93) org.apache.hadoop.hbase.regionserver.AbstractMemStore.(AbstractMemStore.java:83) org.apache.hadoop.hbase.regionserver.DefaultMemStore.(DefaultMemStore.java:79) sun.reflect.GeneratedConstructorAccessor91.newInstance(Unknown Source) sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) java.lang.reflect.Constructor.newInstance(Constructor.java:423) org.apache.hadoop.hbase.util.ReflectionUtils.instantiate(ReflectionUtils.java:55) org.apache.hadoop.hbase.util.ReflectionUtils.newInstance(ReflectionUtils.java:92) org.apache.hadoop.hbase.regionserver.HStore.getMemstore(HStore.java:377) org.apache.hadoop.hbase.regionserver.HStore.(HStore.java:283) org.apache.hadoop.hbase.regionserver.HRegion.instantiateHStore(HRegion.java:6432) org.apache.hadoop.hbase.regionserver.HRegion$1.call(HRegion.java:1140) org.apache.hadoop.hbase.regionserver.HRegion$1.call(HRegion.java:1137) java.util.concurrent.FutureTask.run(FutureTask.java:266) java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) java.util.concurrent.FutureTask.run(FutureTask.java:266) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) 2023-09-06T20:34:19,336 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns2:table2 from hbase:meta 2023-09-06T20:34:19,338 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(387): Removing 'ns2:table2' descriptor. 2023-09-06T20:34:19,339 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=148, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=ns2:table2 2023-09-06T20:34:19,339 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(377): Removing 'ns2:table2' from region states. 2023-09-06T20:34:19,339 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:table2,,1694032452129.66249e4e8d7eb101947abdf29a8225a3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1694032459339"}]},"ts":"9223372036854775807"} 2023-09-06T20:34:19,340 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-06T20:34:19,340 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 66249e4e8d7eb101947abdf29a8225a3, NAME => 'ns2:table2,,1694032452129.66249e4e8d7eb101947abdf29a8225a3.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:34:19,340 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(381): Marking 'ns2:table2' as deleted. 2023-09-06T20:34:19,340 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1694032459340"}]},"ts":"9223372036854775807"} 2023-09-06T20:34:19,342 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1678): Deleted table ns2:table2 state from META 2023-09-06T20:34:19,343 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(127): Finished pid=148, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=ns2:table2 2023-09-06T20:34:19,344 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=148, state=SUCCESS; DeleteTableProcedure table=ns2:table2 in 25 msec 2023-09-06T20:34:19,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=148 2023-09-06T20:34:19,423 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable ns3:table3_restore 2023-09-06T20:34:19,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=149, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns3:table3_restore 2023-09-06T20:34:19,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=149 2023-09-06T20:34:19,427 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032459426"}]},"ts":"1694032459426"} 2023-09-06T20:34:19,428 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=DISABLING in hbase:meta 2023-09-06T20:34:19,430 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(293): Set ns3:table3_restore to state=DISABLING 2023-09-06T20:34:19,431 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:table3_restore, region=5b2405ce5bd5d4fc185b170750bb146c, UNASSIGN}] 2023-09-06T20:34:19,432 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=150, ppid=149, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:table3_restore, region=5b2405ce5bd5d4fc185b170750bb146c, UNASSIGN 2023-09-06T20:34:19,433 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=150 updating hbase:meta row=5b2405ce5bd5d4fc185b170750bb146c, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:19,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE; CloseRegionProcedure 5b2405ce5bd5d4fc185b170750bb146c, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:34:19,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=149 2023-09-06T20:34:19,585 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] handler.UnassignRegionHandler(119): Close 5b2405ce5bd5d4fc185b170750bb146c 2023-09-06T20:34:19,587 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1668): Closing 5b2405ce5bd5d4fc185b170750bb146c, disabling compactions & flushes 2023-09-06T20:34:19,587 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1690): Closing region ns3:table3_restore,,1694032456899.5b2405ce5bd5d4fc185b170750bb146c. 2023-09-06T20:34:19,587 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:table3_restore,,1694032456899.5b2405ce5bd5d4fc185b170750bb146c. 2023-09-06T20:34:19,587 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1778): Acquired close lock on ns3:table3_restore,,1694032456899.5b2405ce5bd5d4fc185b170750bb146c. after waiting 0 ms 2023-09-06T20:34:19,587 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1788): Updates disabled for region ns3:table3_restore,,1694032456899.5b2405ce5bd5d4fc185b170750bb146c. 2023-09-06T20:34:19,590 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/table3_restore/5b2405ce5bd5d4fc185b170750bb146c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-09-06T20:34:19,591 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:34:19,592 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1909): Closed ns3:table3_restore,,1694032456899.5b2405ce5bd5d4fc185b170750bb146c. 2023-09-06T20:34:19,592 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1622): Region close journal for 5b2405ce5bd5d4fc185b170750bb146c: 2023-09-06T20:34:19,593 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] handler.UnassignRegionHandler(163): Closed 5b2405ce5bd5d4fc185b170750bb146c 2023-09-06T20:34:19,594 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=150 updating hbase:meta row=5b2405ce5bd5d4fc185b170750bb146c, regionState=CLOSED 2023-09-06T20:34:19,596 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=151, resume processing ppid=150 2023-09-06T20:34:19,596 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=151, ppid=150, state=SUCCESS; CloseRegionProcedure 5b2405ce5bd5d4fc185b170750bb146c, server=jenkins-hbase3.apache.org,36961,1694032287907 in 161 msec 2023-09-06T20:34:19,597 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=150, resume processing ppid=149 2023-09-06T20:34:19,597 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=150, ppid=149, state=SUCCESS; TransitRegionStateProcedure table=ns3:table3_restore, region=5b2405ce5bd5d4fc185b170750bb146c, UNASSIGN in 165 msec 2023-09-06T20:34:19,597 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032459597"}]},"ts":"1694032459597"} 2023-09-06T20:34:19,598 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=DISABLED in hbase:meta 2023-09-06T20:34:19,600 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set ns3:table3_restore to state=DISABLED 2023-09-06T20:34:19,601 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=149, state=SUCCESS; DisableTableProcedure table=ns3:table3_restore in 177 msec 2023-09-06T20:34:19,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=149 2023-09-06T20:34:19,729 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$5(2448): Client=jenkins//172.31.12.81 delete ns3:table3_restore 2023-09-06T20:34:19,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=152, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=ns3:table3_restore 2023-09-06T20:34:19,732 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=152, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-09-06T20:34:19,732 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=152, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-09-06T20:34:19,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=152 2023-09-06T20:34:19,734 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/table3_restore/5b2405ce5bd5d4fc185b170750bb146c 2023-09-06T20:34:19,735 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/table3_restore/5b2405ce5bd5d4fc185b170750bb146c/f, FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/table3_restore/5b2405ce5bd5d4fc185b170750bb146c/recovered.edits] 2023-09-06T20:34:19,739 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/table3_restore/5b2405ce5bd5d4fc185b170750bb146c/recovered.edits/4.seqid to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/archive/data/ns3/table3_restore/5b2405ce5bd5d4fc185b170750bb146c/recovered.edits/4.seqid 2023-09-06T20:34:19,740 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/table3_restore/5b2405ce5bd5d4fc185b170750bb146c 2023-09-06T20:34:19,740 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(293): Archived ns3:table3_restore regions 2023-09-06T20:34:19,742 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=152, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-09-06T20:34:19,743 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns3:table3_restore from hbase:meta 2023-09-06T20:34:19,745 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(387): Removing 'ns3:table3_restore' descriptor. 2023-09-06T20:34:19,745 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=152, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-09-06T20:34:19,745 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(377): Removing 'ns3:table3_restore' from region states. 2023-09-06T20:34:19,746 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:table3_restore,,1694032456899.5b2405ce5bd5d4fc185b170750bb146c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1694032459745"}]},"ts":"9223372036854775807"} 2023-09-06T20:34:19,747 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-06T20:34:19,747 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 5b2405ce5bd5d4fc185b170750bb146c, NAME => 'ns3:table3_restore,,1694032456899.5b2405ce5bd5d4fc185b170750bb146c.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:34:19,747 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(381): Marking 'ns3:table3_restore' as deleted. 2023-09-06T20:34:19,747 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1694032459747"}]},"ts":"9223372036854775807"} 2023-09-06T20:34:19,748 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table ns3:table3_restore state from META 2023-09-06T20:34:19,750 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(127): Finished pid=152, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-09-06T20:34:19,750 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=152, state=SUCCESS; DeleteTableProcedure table=ns3:table3_restore in 21 msec 2023-09-06T20:34:19,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=152 2023-09-06T20:34:19,851 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultipleCommand Thread=604 (was 605), OpenFileDescriptor=679 (was 683), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=149 (was 151), ProcessCount=167 (was 167), AvailableMemoryMB=3327 (was 3532) 2023-09-06T20:34:19,851 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(130): Thread=604 is superior to 500 2023-09-06T20:34:19,866 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingleDNE Thread=604, OpenFileDescriptor=679, MaxFileDescriptor=60000, SystemLoadAverage=149, ProcessCount=167, AvailableMemoryMB=3326 2023-09-06T20:34:19,866 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(130): Thread=604 is superior to 500 2023-09-06T20:34:19,866 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] backup.TestFullRestore(260): test restore fails on a single table that does not exist 2023-09-06T20:34:19,872 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f0c6638, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:19,874 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:19,875 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45962, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:19,876 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:34:19,878 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:33104, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:34:19,887 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:34:19,891 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:34:19,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032459891 (current time:1694032459891). 2023-09-06T20:34:19,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:34:19,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:34:19,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@604d368d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:19,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:19,899 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45972, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:19,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:19,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:34:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=153, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:34:19,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 153 2023-09-06T20:34:19,902 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:34:19,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=153 2023-09-06T20:34:19,903 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:34:19,904 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:34:20,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=153 2023-09-06T20:34:20,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=153 2023-09-06T20:34:20,310 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:34:20,310 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281}] 2023-09-06T20:34:20,312 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=154, ppid=153, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 2023-09-06T20:34:20,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=154 2023-09-06T20:34:20,463 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:34:20,463 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HRegion(2819): Flushing cf63a85c53279bc94811b50578f31281 2/2 column families, dataSize=1.50 KB heapSize=2.73 KB 2023-09-06T20:34:20,467 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/5c1aa6dd12864ef4a9f61d102553701d is 171, key is trslm:hdfs://localhost:39327/backupUT\x00ns2:test-16940322964961/meta:log-roll-map/1694032451253/Put/seqid=0 2023-09-06T20:34:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=153 2023-09-06T20:34:20,871 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=717 B at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/5c1aa6dd12864ef4a9f61d102553701d 2023-09-06T20:34:20,879 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/df36c3270e704eb2966dff949ba7f613 is 524, key is session:backup_1694032439475/session:context/1694032452082/Put/seqid=0 2023-09-06T20:34:21,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=153 2023-09-06T20:34:21,283 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=820 B at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/df36c3270e704eb2966dff949ba7f613 2023-09-06T20:34:21,288 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/5c1aa6dd12864ef4a9f61d102553701d as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/5c1aa6dd12864ef4a9f61d102553701d 2023-09-06T20:34:21,292 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/5c1aa6dd12864ef4a9f61d102553701d, entries=6, sequenceid=116, filesize=5.8 K 2023-09-06T20:34:21,293 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/df36c3270e704eb2966dff949ba7f613 as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/df36c3270e704eb2966dff949ba7f613 2023-09-06T20:34:21,297 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/df36c3270e704eb2966dff949ba7f613, entries=2, sequenceid=116, filesize=5.5 K 2023-09-06T20:34:21,298 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HRegion(3022): Finished flush of dataSize ~1.50 KB/1537, heapSize ~2.70 KB/2760, currentSize=0 B/0 for cf63a85c53279bc94811b50578f31281 in 835ms, sequenceid=116, compaction requested=true 2023-09-06T20:34:21,298 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HRegion(2520): Flush status journal for cf63a85c53279bc94811b50578f31281: 2023-09-06T20:34:21,298 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. for snapshot_backup_system completed. 2023-09-06T20:34:21,298 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:34:21,298 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:34:21,298 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/0379b1e1f59e40d2ac2991b2ae6b86ce, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/dd7edbeade334fa7895ff079e0fc3472, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/78eec26cfa0242e48353b19a4a00b7d5, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/faffbcb72f8649bcad705936a3105fd4, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/01fb4a2eb68c4823807811190aa11f8b, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/b14bcdd382934779b0fa80ae042378aa, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/b7aca3aa47f24516bd2686bfd6b9193d, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/48f5b68b63f84de7a7c1f4f08993f64f, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/3ff5f0a110934ea19f13a9dc5a7d2318, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/5c1aa6dd12864ef4a9f61d102553701d] hfiles 2023-09-06T20:34:21,298 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (1/10): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/0379b1e1f59e40d2ac2991b2ae6b86ce for snapshot=snapshot_backup_system 2023-09-06T20:34:21,298 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (2/10): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/dd7edbeade334fa7895ff079e0fc3472 for snapshot=snapshot_backup_system 2023-09-06T20:34:21,298 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (3/10): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/78eec26cfa0242e48353b19a4a00b7d5 for snapshot=snapshot_backup_system 2023-09-06T20:34:21,298 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (4/10): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/faffbcb72f8649bcad705936a3105fd4 for snapshot=snapshot_backup_system 2023-09-06T20:34:21,298 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (5/10): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/01fb4a2eb68c4823807811190aa11f8b for snapshot=snapshot_backup_system 2023-09-06T20:34:21,298 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (6/10): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/b14bcdd382934779b0fa80ae042378aa for snapshot=snapshot_backup_system 2023-09-06T20:34:21,298 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (7/10): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/b7aca3aa47f24516bd2686bfd6b9193d for snapshot=snapshot_backup_system 2023-09-06T20:34:21,298 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (8/10): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/48f5b68b63f84de7a7c1f4f08993f64f for snapshot=snapshot_backup_system 2023-09-06T20:34:21,298 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (9/10): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/3ff5f0a110934ea19f13a9dc5a7d2318 for snapshot=snapshot_backup_system 2023-09-06T20:34:21,298 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (10/10): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/5c1aa6dd12864ef4a9f61d102553701d for snapshot=snapshot_backup_system 2023-09-06T20:34:21,299 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/4ced064cdbd34bd6b594c76f049fa4ee, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/baca9a30045a4dabad9b3b78a8fd6c86, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/bcfde09dc46641bebf14d756f321d0af, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/2dc68dbd9f3749a1a2d9d7333fa4771d, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/6da826e7b47e41daa41e268fb4e951b2, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/849e36507ab34f85a508bffbc6e215d3, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/8bd33efd4a8e44d1b5b9f6a66a166990, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/13357027918440549abec030e3035318, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/a2912410595b4f2d8ed499b1de1206c9, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/c1202b8538934be1bed1ea11b0457324, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/df36c3270e704eb2966dff949ba7f613] hfiles 2023-09-06T20:34:21,299 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (1/11): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/4ced064cdbd34bd6b594c76f049fa4ee for snapshot=snapshot_backup_system 2023-09-06T20:34:21,299 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (2/11): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/baca9a30045a4dabad9b3b78a8fd6c86 for snapshot=snapshot_backup_system 2023-09-06T20:34:21,299 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (3/11): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/bcfde09dc46641bebf14d756f321d0af for snapshot=snapshot_backup_system 2023-09-06T20:34:21,299 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (4/11): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/2dc68dbd9f3749a1a2d9d7333fa4771d for snapshot=snapshot_backup_system 2023-09-06T20:34:21,299 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (5/11): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/6da826e7b47e41daa41e268fb4e951b2 for snapshot=snapshot_backup_system 2023-09-06T20:34:21,299 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (6/11): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/849e36507ab34f85a508bffbc6e215d3 for snapshot=snapshot_backup_system 2023-09-06T20:34:21,299 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (7/11): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/8bd33efd4a8e44d1b5b9f6a66a166990 for snapshot=snapshot_backup_system 2023-09-06T20:34:21,299 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (8/11): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/13357027918440549abec030e3035318 for snapshot=snapshot_backup_system 2023-09-06T20:34:21,299 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (9/11): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/a2912410595b4f2d8ed499b1de1206c9 for snapshot=snapshot_backup_system 2023-09-06T20:34:21,299 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (10/11): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/c1202b8538934be1bed1ea11b0457324 for snapshot=snapshot_backup_system 2023-09-06T20:34:21,299 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (11/11): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/df36c3270e704eb2966dff949ba7f613 for snapshot=snapshot_backup_system 2023-09-06T20:34:21,703 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:34:21,704 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=154 2023-09-06T20:34:21,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=154 2023-09-06T20:34:21,704 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region cf63a85c53279bc94811b50578f31281 2023-09-06T20:34:21,705 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=154, ppid=153, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 2023-09-06T20:34:21,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=154, resume processing ppid=153 2023-09-06T20:34:21,707 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:34:21,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=154, ppid=153, state=SUCCESS; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 in 1.3960 sec 2023-09-06T20:34:21,708 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:34:21,709 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:34:21,709 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:34:21,709 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:34:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=153 2023-09-06T20:34:22,121 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:34:22,133 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:34:22,133 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:34:22,134 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:34:22,134 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 153 2023-09-06T20:34:22,135 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=153, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2340 sec 2023-09-06T20:34:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=153 2023-09-06T20:34:24,009 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(115): Backup backup_1694032459872 started at 1694032464009. 2023-09-06T20:34:24,010 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(119): Backup session backup_1694032459872 has been started. 2023-09-06T20:34:24,018 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-09-06T20:34:24,018 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-09-06T20:34:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-09-06T20:34:24,019 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-09-06T20:34:24,019 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-09-06T20:34:24,020 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-09-06T20:34:24,020 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-09-06T20:34:24,020 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:34:24,022 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:24,022 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:34:24,022 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:34:24,022 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-09-06T20:34:24,022 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-09-06T20:34:24,023 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-09-06T20:34:24,023 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:34:24,023 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-09-06T20:34:24,023 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-09-06T20:34:24,023 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-09-06T20:34:24,023 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-09-06T20:34:24,023 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-09-06T20:34:24,024 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-09-06T20:34:24,024 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-09-06T20:34:24,024 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,36961,1694032287907' joining acquired barrier for procedure (rolllog) in zk 2023-09-06T20:34:24,026 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-09-06T20:34:24,026 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:24,026 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:34:24,026 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:34:24,027 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-09-06T20:34:24,027 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:34:24,027 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:34:24,027 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:24,028 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:24,028 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:34:24,028 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,36961,1694032287907' joining acquired barrier for procedure 'rolllog' on coordinator 2023-09-06T20:34:24,028 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@73c42c95[Count = 0] remaining members to acquire global barrier 2023-09-06T20:34:24,028 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-09-06T20:34:24,028 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-09-06T20:34:24,029 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-09-06T20:34:24,029 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-09-06T20:34:24,029 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-09-06T20:34:24,030 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-09-06T20:34:24,030 DEBUG [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:24,030 INFO [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1694032441662 highest: 1694032441662 on jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:24,030 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta:.meta(num 1694032441644) roll requested 2023-09-06T20:34:24,030 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-09-06T20:34:24,041 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032464030.meta, exclude list is [], retry=0 2023-09-06T20:34:24,045 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK] 2023-09-06T20:34:24,047 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032441644.meta with entries=24, filesize=5.77 KB; new WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032464030.meta 2023-09-06T20:34:24,047 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK]] 2023-09-06T20:34:24,047 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032441644.meta is not closed yet, will try archiving it next time 2023-09-06T20:34:24,047 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36961%2C1694032287907:(num 1694032441662) roll requested 2023-09-06T20:34:24,059 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032464047, exclude list is [], retry=0 2023-09-06T20:34:24,063 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK] 2023-09-06T20:34:24,067 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032441662 with entries=15, filesize=4.05 KB; new WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032464047 2023-09-06T20:34:24,067 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK]] 2023-09-06T20:34:24,067 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032441662 is not closed yet, will try archiving it next time 2023-09-06T20:34:24,070 DEBUG [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-09-06T20:34:24,070 INFO [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1694032464047 on jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:24,081 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-09-06T20:34:24,081 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,36961,1694032287907' in zk 2023-09-06T20:34:24,085 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-09-06T20:34:24,085 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:34:24,085 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:24,085 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-09-06T20:34:24,086 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:34:24,086 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:34:24,086 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:34:24,087 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:34:24,087 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:24,087 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:24,088 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:34:24,088 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:24,088 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:24,089 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,36961,1694032287907': 2023-09-06T20:34:24,089 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,36961,1694032287907' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-09-06T20:34:24,089 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-09-06T20:34:24,089 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-09-06T20:34:24,089 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-09-06T20:34:24,089 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-09-06T20:34:24,089 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-09-06T20:34:24,091 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-09-06T20:34:24,091 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:34:24,091 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:34:24,091 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-09-06T20:34:24,091 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:34:24,092 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:34:24,092 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:34:24,092 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:34:24,092 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:34:24,092 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:24,093 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:34:24,093 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:24,093 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:24,094 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:34:24,094 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:24,094 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:24,098 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:34:24,098 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:34:24,098 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:34:24,098 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-09-06T20:34:24,098 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-09-06T20:34:24,099 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:34:24,099 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:34:24,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-09-06T20:34:24,203 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1694032464202_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } 2023-09-06T20:34:24,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032464203 (current time:1694032464203). 2023-09-06T20:34:24,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:34:24,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1694032464202_default_test-1694032296496 VERSION not specified, setting to 2 2023-09-06T20:34:24,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@354fbc0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:24,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:24,220 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39824, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:24,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:24,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:34:24,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=155, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1694032464202_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } 2023-09-06T20:34:24,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1694032464202_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 }, snapshot procedure id = 155 2023-09-06T20:34:24,224 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1694032464202_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:34:24,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=155 2023-09-06T20:34:24,225 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1694032464202_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:34:24,227 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1694032464202_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:34:24,237 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1694032464202_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:34:24,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE; SnapshotRegionProcedure 1f072a702e0732d4b296550713fb6262}] 2023-09-06T20:34:24,239 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=156, ppid=155, state=RUNNABLE; SnapshotRegionProcedure 1f072a702e0732d4b296550713fb6262 2023-09-06T20:34:24,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=155 2023-09-06T20:34:24,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=156 2023-09-06T20:34:24,389 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1694032296496,,1694032379919.1f072a702e0732d4b296550713fb6262. 2023-09-06T20:34:24,390 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] regionserver.HRegion(2520): Flush status journal for 1f072a702e0732d4b296550713fb6262: 2023-09-06T20:34:24,390 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1694032296496,,1694032379919.1f072a702e0732d4b296550713fb6262. for snapshot_1694032464202_default_test-1694032296496 completed. 2023-09-06T20:34:24,390 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] snapshot.SnapshotManifest(240): Storing 'test-1694032296496,,1694032379919.1f072a702e0732d4b296550713fb6262.' region-info for snapshot=snapshot_1694032464202_default_test-1694032296496 2023-09-06T20:34:24,390 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:34:24,390 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/1f072a702e0732d4b296550713fb6262/f/8c33b1e0971346619bc1707bbedeefce_SeqId_4_] hfiles 2023-09-06T20:34:24,390 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/1f072a702e0732d4b296550713fb6262/f/8c33b1e0971346619bc1707bbedeefce_SeqId_4_ for snapshot=snapshot_1694032464202_default_test-1694032296496 2023-09-06T20:34:24,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=155 2023-09-06T20:34:24,681 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:34:24,795 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1694032296496,,1694032379919.1f072a702e0732d4b296550713fb6262. 2023-09-06T20:34:24,795 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=156 2023-09-06T20:34:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=156 2023-09-06T20:34:24,796 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1694032464202_default_test-1694032296496 on region 1f072a702e0732d4b296550713fb6262 2023-09-06T20:34:24,797 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=156, ppid=155, state=RUNNABLE; SnapshotRegionProcedure 1f072a702e0732d4b296550713fb6262 2023-09-06T20:34:24,799 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=156, resume processing ppid=155 2023-09-06T20:34:24,799 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1694032464202_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:34:24,799 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=156, ppid=155, state=SUCCESS; SnapshotRegionProcedure 1f072a702e0732d4b296550713fb6262 in 561 msec 2023-09-06T20:34:24,800 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1694032464202_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:34:24,800 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1694032464202_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:34:24,800 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1694032464202_default_test-1694032296496 2023-09-06T20:34:24,801 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032464202_default_test-1694032296496 2023-09-06T20:34:24,808 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1694032464202_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:34:24,813 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1694032464202_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:34:24,813 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032464202_default_test-1694032296496 to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032464202_default_test-1694032296496 2023-09-06T20:34:24,814 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1694032464202_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:34:24,814 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1694032464202_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 }, snapshot procedure id = 155 2023-09-06T20:34:24,815 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=155, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1694032464202_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } in 593 msec 2023-09-06T20:34:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=155 2023-09-06T20:34:24,831 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1694032459872 2023-09-06T20:34:24,831 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-09-06T20:34:24,831 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-09-06T20:34:24,831 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1694032459872_test-1694032296496 2023-09-06T20:34:24,831 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1694032464202_default_test-1694032296496 to hdfs://localhost:39327/backupUT/backup_1694032459872/default/test-1694032296496/ 2023-09-06T20:34:24,832 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-09-06T20:34:24,854 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:34:24,854 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37419,1694032284546_2090846263_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:39327/backupUT/backup_1694032459872/default/test-1694032296496, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:39327/backupUT/backup_1694032459872/default/test-1694032296496/.hbase-snapshot/.tmp/snapshot_1694032464202_default_test-1694032296496 2023-09-06T20:34:24,854 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1, snapshotDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032464202_default_test-1694032296496. 2023-09-06T20:34:24,861 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032464202_default_test-1694032296496 to hdfs://localhost:39327/backupUT/backup_1694032459872/default/test-1694032296496/.hbase-snapshot/.tmp/snapshot_1694032464202_default_test-1694032296496 2023-09-06T20:34:25,276 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:25,276 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:25,277 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:25,277 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:25,747 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_table2 2023-09-06T20:34:26,251 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop-2061635464880093383.jar 2023-09-06T20:34:26,252 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:26,252 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:26,253 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:26,253 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:26,254 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:26,255 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:26,255 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:26,256 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:26,256 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:26,257 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:34:26,258 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:34:26,259 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:34:26,259 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:34:26,260 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:34:26,261 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:34:26,261 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:34:26,262 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:34:26,262 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:34:26,263 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:34:26,264 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:34:26,264 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:34:26,265 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:26,266 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:26,267 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:34:26,267 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:26,268 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:26,268 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:34:26,269 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:34:26,317 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1694032464202_default_test-1694032296496' hfile list 2023-09-06T20:34:26,319 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-09-06T20:34:26,389 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032466354/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:34:26,389 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032466354/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:34:26,463 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-09-06T20:34:26,487 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/1f072a702e0732d4b296550713fb6262/f/8c33b1e0971346619bc1707bbedeefce_SeqId_4_ output=hdfs://localhost:39327/backupUT/backup_1694032459872/default/test-1694032296496/archive/data/default/test-1694032296496/1f072a702e0732d4b296550713fb6262/f/8c33b1e0971346619bc1707bbedeefce_SeqId_4_ 2023-09-06T20:34:26,487 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8725 (8.5 K) time=0sec 8.321M/sec 2023-09-06T20:34:27,443 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-09-06T20:34:27,444 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-09-06T20:34:27,449 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1694032464202_default_test-1694032296496 2023-09-06T20:34:27,449 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1694032464202_default_test-1694032296496 finished. 2023-09-06T20:34:27,449 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1014): test-1694032296496 2023-09-06T20:34:27,455 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1694032459872 2023-09-06T20:34:27,455 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-09-06T20:34:27,860 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:39327/backupUT/backup_1694032459872/.backup.manifest 2023-09-06T20:34:27,860 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(358): Backup backup_1694032459872 finished: type=FULL,tablelist=test-1694032296496,targetRootDir=hdfs://localhost:39327/backupUT,startts=1694032464009,completets=1694032467455,bytescopied=0 2023-09-06T20:34:27,860 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-09-06T20:34:27,860 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1694032464202_default_test-1694032296496 2023-09-06T20:34:27,861 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1694032464202_default_test-1694032296496" 2023-09-06T20:34:27,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1694032464202_default_test-1694032296496 2023-09-06T20:34:27,864 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1694032464202_default_test-1694032296496 for backup backup_1694032459872 succeeded. 2023-09-06T20:34:27,865 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:34:27,869 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:34:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:34:27,872 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:34:27,873 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:34:27,878 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(376): Backup backup_1694032459872 completed. 2023-09-06T20:34:27,878 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:27,888 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] backup.TestFullRestore(265): backup complete 2023-09-06T20:34:27,889 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(101): HBase table table1 does not exist. It will be created during restore process 2023-09-06T20:34:27,890 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:39327/backupUT/backup_1694032459872/.backup.manifest 2023-09-06T20:34:27,891 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032459872/.backup.manifest 2023-09-06T20:34:27,892 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:39327/backupUT/backup_1694032459872/.backup.manifest 2023-09-06T20:34:27,894 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032459872/.backup.manifest 2023-09-06T20:34:27,894 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(148): Restoring 'faketable' to 'table1' from full backup image hdfs://localhost:39327/backupUT/backup_1694032459872/default/faketable 2023-09-06T20:34:27,913 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingleDNE Thread=603 (was 604), OpenFileDescriptor=695 (was 679) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=161 (was 149) - SystemLoadAverage LEAK? -, ProcessCount=167 (was 167), AvailableMemoryMB=3183 (was 3326) 2023-09-06T20:34:27,913 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(130): Thread=603 is superior to 500 2023-09-06T20:34:27,929 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingleCommand Thread=603, OpenFileDescriptor=695, MaxFileDescriptor=60000, SystemLoadAverage=161, ProcessCount=167, AvailableMemoryMB=3182 2023-09-06T20:34:27,930 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(130): Thread=603 is superior to 500 2023-09-06T20:34:27,930 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] backup.TestFullRestore(76): test full restore on a single table empty table: command-line 2023-09-06T20:34:27,935 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c73ddc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:27,937 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:27,938 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39826, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:27,939 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:34:27,941 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:47288, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:34:27,952 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:34:27,956 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:34:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032467956 (current time:1694032467956). 2023-09-06T20:34:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:34:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:34:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38515e9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:27,962 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39830, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:34:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=157, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:34:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 157 2023-09-06T20:34:27,965 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:34:27,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=157 2023-09-06T20:34:27,966 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:34:27,967 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:34:28,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=157 2023-09-06T20:34:28,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=157 2023-09-06T20:34:28,373 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:34:28,373 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281}] 2023-09-06T20:34:28,375 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=158, ppid=157, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 2023-09-06T20:34:28,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=158 2023-09-06T20:34:28,527 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:34:28,527 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HRegion(2819): Flushing cf63a85c53279bc94811b50578f31281 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-09-06T20:34:28,530 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/ba7080b60f13447ca7b0f338dd848e23 is 169, key is trslm:hdfs://localhost:39327/backupUT\x00test-1694032296496/meta:log-roll-map/1694032467451/Put/seqid=0 2023-09-06T20:34:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=157 2023-09-06T20:34:28,934 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/ba7080b60f13447ca7b0f338dd848e23 2023-09-06T20:34:28,943 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/060ea6a84a9a417fabf8ead571bb96d5 is 452, key is session:backup_1694032459872/session:context/1694032467873/Put/seqid=0 2023-09-06T20:34:28,948 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/060ea6a84a9a417fabf8ead571bb96d5 2023-09-06T20:34:28,954 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/ba7080b60f13447ca7b0f338dd848e23 as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/ba7080b60f13447ca7b0f338dd848e23 2023-09-06T20:34:28,959 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/ba7080b60f13447ca7b0f338dd848e23, entries=4, sequenceid=127, filesize=5.5 K 2023-09-06T20:34:28,960 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/060ea6a84a9a417fabf8ead571bb96d5 as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/060ea6a84a9a417fabf8ead571bb96d5 2023-09-06T20:34:28,970 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/060ea6a84a9a417fabf8ead571bb96d5, entries=2, sequenceid=127, filesize=5.4 K 2023-09-06T20:34:28,971 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for cf63a85c53279bc94811b50578f31281 in 444ms, sequenceid=127, compaction requested=true 2023-09-06T20:34:28,971 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HRegion(2520): Flush status journal for cf63a85c53279bc94811b50578f31281: 2023-09-06T20:34:28,971 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. for snapshot_backup_system completed. 2023-09-06T20:34:28,971 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:34:28,971 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:34:28,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/0379b1e1f59e40d2ac2991b2ae6b86ce, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/dd7edbeade334fa7895ff079e0fc3472, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/78eec26cfa0242e48353b19a4a00b7d5, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/faffbcb72f8649bcad705936a3105fd4, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/01fb4a2eb68c4823807811190aa11f8b, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/b14bcdd382934779b0fa80ae042378aa, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/b7aca3aa47f24516bd2686bfd6b9193d, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/48f5b68b63f84de7a7c1f4f08993f64f, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/3ff5f0a110934ea19f13a9dc5a7d2318, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/5c1aa6dd12864ef4a9f61d102553701d, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/ba7080b60f13447ca7b0f338dd848e23] hfiles 2023-09-06T20:34:28,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (1/11): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/0379b1e1f59e40d2ac2991b2ae6b86ce for snapshot=snapshot_backup_system 2023-09-06T20:34:28,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (2/11): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/dd7edbeade334fa7895ff079e0fc3472 for snapshot=snapshot_backup_system 2023-09-06T20:34:28,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (3/11): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/78eec26cfa0242e48353b19a4a00b7d5 for snapshot=snapshot_backup_system 2023-09-06T20:34:28,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (4/11): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/faffbcb72f8649bcad705936a3105fd4 for snapshot=snapshot_backup_system 2023-09-06T20:34:28,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (5/11): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/01fb4a2eb68c4823807811190aa11f8b for snapshot=snapshot_backup_system 2023-09-06T20:34:28,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (6/11): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/b14bcdd382934779b0fa80ae042378aa for snapshot=snapshot_backup_system 2023-09-06T20:34:28,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (7/11): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/b7aca3aa47f24516bd2686bfd6b9193d for snapshot=snapshot_backup_system 2023-09-06T20:34:28,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (8/11): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/48f5b68b63f84de7a7c1f4f08993f64f for snapshot=snapshot_backup_system 2023-09-06T20:34:28,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (9/11): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/3ff5f0a110934ea19f13a9dc5a7d2318 for snapshot=snapshot_backup_system 2023-09-06T20:34:28,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (10/11): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/5c1aa6dd12864ef4a9f61d102553701d for snapshot=snapshot_backup_system 2023-09-06T20:34:28,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (11/11): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/ba7080b60f13447ca7b0f338dd848e23 for snapshot=snapshot_backup_system 2023-09-06T20:34:28,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/4ced064cdbd34bd6b594c76f049fa4ee, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/baca9a30045a4dabad9b3b78a8fd6c86, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/bcfde09dc46641bebf14d756f321d0af, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/2dc68dbd9f3749a1a2d9d7333fa4771d, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/6da826e7b47e41daa41e268fb4e951b2, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/849e36507ab34f85a508bffbc6e215d3, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/8bd33efd4a8e44d1b5b9f6a66a166990, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/13357027918440549abec030e3035318, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/a2912410595b4f2d8ed499b1de1206c9, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/c1202b8538934be1bed1ea11b0457324, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/df36c3270e704eb2966dff949ba7f613, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/060ea6a84a9a417fabf8ead571bb96d5] hfiles 2023-09-06T20:34:28,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (1/12): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/4ced064cdbd34bd6b594c76f049fa4ee for snapshot=snapshot_backup_system 2023-09-06T20:34:28,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (2/12): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/baca9a30045a4dabad9b3b78a8fd6c86 for snapshot=snapshot_backup_system 2023-09-06T20:34:28,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (3/12): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/bcfde09dc46641bebf14d756f321d0af for snapshot=snapshot_backup_system 2023-09-06T20:34:28,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (4/12): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/2dc68dbd9f3749a1a2d9d7333fa4771d for snapshot=snapshot_backup_system 2023-09-06T20:34:28,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (5/12): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/6da826e7b47e41daa41e268fb4e951b2 for snapshot=snapshot_backup_system 2023-09-06T20:34:28,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (6/12): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/849e36507ab34f85a508bffbc6e215d3 for snapshot=snapshot_backup_system 2023-09-06T20:34:28,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (7/12): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/8bd33efd4a8e44d1b5b9f6a66a166990 for snapshot=snapshot_backup_system 2023-09-06T20:34:28,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (8/12): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/13357027918440549abec030e3035318 for snapshot=snapshot_backup_system 2023-09-06T20:34:28,973 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (9/12): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/a2912410595b4f2d8ed499b1de1206c9 for snapshot=snapshot_backup_system 2023-09-06T20:34:28,973 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (10/12): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/c1202b8538934be1bed1ea11b0457324 for snapshot=snapshot_backup_system 2023-09-06T20:34:28,973 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (11/12): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/df36c3270e704eb2966dff949ba7f613 for snapshot=snapshot_backup_system 2023-09-06T20:34:28,973 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (12/12): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/060ea6a84a9a417fabf8ead571bb96d5 for snapshot=snapshot_backup_system 2023-09-06T20:34:29,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=157 2023-09-06T20:34:29,387 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:34:29,387 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=158 2023-09-06T20:34:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=158 2023-09-06T20:34:29,388 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region cf63a85c53279bc94811b50578f31281 2023-09-06T20:34:29,389 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=158, ppid=157, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 2023-09-06T20:34:29,391 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=158, resume processing ppid=157 2023-09-06T20:34:29,391 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:34:29,391 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=158, ppid=157, state=SUCCESS; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 in 1.0170 sec 2023-09-06T20:34:29,392 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:34:29,392 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:34:29,392 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:34:29,393 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:34:29,802 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:34:29,814 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:34:29,814 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:34:29,815 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:34:29,815 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 157 2023-09-06T20:34:29,816 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=157, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8520 sec 2023-09-06T20:34:29,900 DEBUG [regionserver/jenkins-hbase3:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.6923076923076923, tune throughput to 84.62 MB/second 2023-09-06T20:34:30,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=157 2023-09-06T20:34:30,070 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(115): Backup backup_1694032467935 started at 1694032470070. 2023-09-06T20:34:30,071 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(119): Backup session backup_1694032467935 has been started. 2023-09-06T20:34:30,076 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-09-06T20:34:30,077 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-09-06T20:34:30,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-09-06T20:34:30,077 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-09-06T20:34:30,077 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-09-06T20:34:30,078 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-09-06T20:34:30,078 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-09-06T20:34:30,078 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:34:30,081 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:30,081 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:34:30,081 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:34:30,081 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-09-06T20:34:30,081 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-09-06T20:34:30,082 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-09-06T20:34:30,082 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:34:30,082 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-09-06T20:34:30,082 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-09-06T20:34:30,082 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-09-06T20:34:30,082 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-09-06T20:34:30,082 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-09-06T20:34:30,083 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-09-06T20:34:30,083 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-09-06T20:34:30,083 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,36961,1694032287907' joining acquired barrier for procedure (rolllog) in zk 2023-09-06T20:34:30,085 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-09-06T20:34:30,085 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:30,085 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:34:30,085 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:34:30,085 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-09-06T20:34:30,085 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:34:30,085 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:34:30,086 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:30,086 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:30,086 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:34:30,086 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,36961,1694032287907' joining acquired barrier for procedure 'rolllog' on coordinator 2023-09-06T20:34:30,086 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@7143a812[Count = 0] remaining members to acquire global barrier 2023-09-06T20:34:30,086 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-09-06T20:34:30,086 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-09-06T20:34:30,087 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-09-06T20:34:30,088 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-09-06T20:34:30,088 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-09-06T20:34:30,088 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-09-06T20:34:30,088 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-09-06T20:34:30,088 DEBUG [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:30,088 INFO [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1694032464047 highest: 1694032464047 on jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:30,088 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta:.meta(num 1694032464030) roll requested 2023-09-06T20:34:30,099 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032470088.meta, exclude list is [], retry=0 2023-09-06T20:34:30,102 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK] 2023-09-06T20:34:30,104 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032464030.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032470088.meta 2023-09-06T20:34:30,104 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK]] 2023-09-06T20:34:30,104 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032464030.meta is not closed yet, will try archiving it next time 2023-09-06T20:34:30,104 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36961%2C1694032287907:(num 1694032464047) roll requested 2023-09-06T20:34:30,106 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032464030.meta to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/oldWALs/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032464030.meta 2023-09-06T20:34:30,115 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032470104, exclude list is [], retry=0 2023-09-06T20:34:30,118 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK] 2023-09-06T20:34:30,119 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032464047 with entries=10, filesize=2.31 KB; new WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032470104 2023-09-06T20:34:30,119 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK]] 2023-09-06T20:34:30,119 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032464047 is not closed yet, will try archiving it next time 2023-09-06T20:34:30,119 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032441662 to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/oldWALs/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032441662 2023-09-06T20:34:30,128 DEBUG [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-09-06T20:34:30,128 INFO [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1694032470104 on jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:30,135 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-09-06T20:34:30,135 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,36961,1694032287907' in zk 2023-09-06T20:34:30,138 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-09-06T20:34:30,138 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:30,138 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:34:30,138 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:34:30,138 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-09-06T20:34:30,138 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:34:30,139 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:34:30,139 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:34:30,140 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:30,140 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:30,140 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:34:30,140 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:30,140 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:30,141 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,36961,1694032287907': 2023-09-06T20:34:30,141 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,36961,1694032287907' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-09-06T20:34:30,141 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-09-06T20:34:30,141 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-09-06T20:34:30,141 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-09-06T20:34:30,141 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-09-06T20:34:30,141 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-09-06T20:34:30,144 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-09-06T20:34:30,144 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:34:30,144 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:34:30,144 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-09-06T20:34:30,144 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:34:30,144 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:34:30,144 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:34:30,144 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:34:30,144 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:30,144 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:34:30,145 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:34:30,145 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:30,145 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:30,145 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:34:30,146 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:30,146 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:30,156 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:34:30,157 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:34:30,157 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:34:30,157 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-09-06T20:34:30,157 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-09-06T20:34:30,157 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:34:30,157 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:34:30,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-09-06T20:34:30,261 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1694032470260_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } 2023-09-06T20:34:30,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032470261 (current time:1694032470261). 2023-09-06T20:34:30,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:34:30,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1694032470260_default_test-1694032296496 VERSION not specified, setting to 2 2023-09-06T20:34:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29c3b2ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:30,269 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39870, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:34:30,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=159, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1694032470260_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } 2023-09-06T20:34:30,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1694032470260_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 }, snapshot procedure id = 159 2023-09-06T20:34:30,273 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1694032470260_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:34:30,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=159 2023-09-06T20:34:30,273 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1694032470260_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:34:30,275 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1694032470260_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:34:30,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=159 2023-09-06T20:34:30,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=159 2023-09-06T20:34:30,680 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1694032470260_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:34:30,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE; SnapshotRegionProcedure 1f072a702e0732d4b296550713fb6262}] 2023-09-06T20:34:30,682 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=160, ppid=159, state=RUNNABLE; SnapshotRegionProcedure 1f072a702e0732d4b296550713fb6262 2023-09-06T20:34:30,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=160 2023-09-06T20:34:30,834 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1694032296496,,1694032379919.1f072a702e0732d4b296550713fb6262. 2023-09-06T20:34:30,834 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] regionserver.HRegion(2520): Flush status journal for 1f072a702e0732d4b296550713fb6262: 2023-09-06T20:34:30,834 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1694032296496,,1694032379919.1f072a702e0732d4b296550713fb6262. for snapshot_1694032470260_default_test-1694032296496 completed. 2023-09-06T20:34:30,834 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] snapshot.SnapshotManifest(240): Storing 'test-1694032296496,,1694032379919.1f072a702e0732d4b296550713fb6262.' region-info for snapshot=snapshot_1694032470260_default_test-1694032296496 2023-09-06T20:34:30,834 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:34:30,834 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/1f072a702e0732d4b296550713fb6262/f/8c33b1e0971346619bc1707bbedeefce_SeqId_4_] hfiles 2023-09-06T20:34:30,834 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/1f072a702e0732d4b296550713fb6262/f/8c33b1e0971346619bc1707bbedeefce_SeqId_4_ for snapshot=snapshot_1694032470260_default_test-1694032296496 2023-09-06T20:34:30,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=159 2023-09-06T20:34:31,240 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1694032296496,,1694032379919.1f072a702e0732d4b296550713fb6262. 2023-09-06T20:34:31,240 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=160 2023-09-06T20:34:31,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=160 2023-09-06T20:34:31,241 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1694032470260_default_test-1694032296496 on region 1f072a702e0732d4b296550713fb6262 2023-09-06T20:34:31,242 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=160, ppid=159, state=RUNNABLE; SnapshotRegionProcedure 1f072a702e0732d4b296550713fb6262 2023-09-06T20:34:31,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=160, resume processing ppid=159 2023-09-06T20:34:31,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=160, ppid=159, state=SUCCESS; SnapshotRegionProcedure 1f072a702e0732d4b296550713fb6262 in 563 msec 2023-09-06T20:34:31,244 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1694032470260_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:34:31,244 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1694032470260_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:34:31,245 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1694032470260_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:34:31,245 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1694032470260_default_test-1694032296496 2023-09-06T20:34:31,246 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032470260_default_test-1694032296496 2023-09-06T20:34:31,249 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:34:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=159 2023-09-06T20:34:31,660 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1694032470260_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:34:31,665 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1694032470260_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:34:31,666 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032470260_default_test-1694032296496 to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032470260_default_test-1694032296496 2023-09-06T20:34:31,667 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1694032470260_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:34:31,667 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1694032470260_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 }, snapshot procedure id = 159 2023-09-06T20:34:31,668 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=159, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1694032470260_default_test-1694032296496 table=test-1694032296496 type=FLUSH ttl=0 } in 1.3970 sec 2023-09-06T20:34:32,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=159 2023-09-06T20:34:32,379 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1694032467935 2023-09-06T20:34:32,379 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-09-06T20:34:32,379 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-09-06T20:34:32,379 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1694032467935_test-1694032296496 2023-09-06T20:34:32,379 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1694032470260_default_test-1694032296496 to hdfs://localhost:39327/backupUT/backup_1694032467935/default/test-1694032296496/ 2023-09-06T20:34:32,379 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-09-06T20:34:32,403 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:34:32,403 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37419,1694032284546_-934179455_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:39327/backupUT/backup_1694032467935/default/test-1694032296496, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:39327/backupUT/backup_1694032467935/default/test-1694032296496/.hbase-snapshot/.tmp/snapshot_1694032470260_default_test-1694032296496 2023-09-06T20:34:32,403 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1, snapshotDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032470260_default_test-1694032296496. 2023-09-06T20:34:32,409 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032470260_default_test-1694032296496 to hdfs://localhost:39327/backupUT/backup_1694032467935/default/test-1694032296496/.hbase-snapshot/.tmp/snapshot_1694032470260_default_test-1694032296496 2023-09-06T20:34:32,827 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:32,827 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:32,828 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:32,828 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:33,808 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop-2156692603982886463.jar 2023-09-06T20:34:33,809 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:33,809 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:33,809 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:33,810 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:33,810 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:33,810 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:33,811 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:33,811 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:33,811 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:33,812 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:34:33,813 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:34:33,813 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:34:33,813 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:34:33,814 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:34:33,814 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:34:33,814 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:34:33,815 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:34:33,815 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:34:33,815 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:34:33,816 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:34:33,816 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:34:33,817 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:33,817 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:33,818 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:34:33,818 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:33,818 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:33,819 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:34:33,819 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:34:33,861 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1694032470260_default_test-1694032296496' hfile list 2023-09-06T20:34:33,862 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-09-06T20:34:33,930 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032473897/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:34:33,930 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032473897/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:34:33,984 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-09-06T20:34:34,008 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/1f072a702e0732d4b296550713fb6262/f/8c33b1e0971346619bc1707bbedeefce_SeqId_4_ output=hdfs://localhost:39327/backupUT/backup_1694032467935/default/test-1694032296496/archive/data/default/test-1694032296496/1f072a702e0732d4b296550713fb6262/f/8c33b1e0971346619bc1707bbedeefce_SeqId_4_ 2023-09-06T20:34:34,008 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8725 (8.5 K) time=0sec 8.321M/sec 2023-09-06T20:34:34,960 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-09-06T20:34:34,963 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-09-06T20:34:34,968 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1694032470260_default_test-1694032296496 2023-09-06T20:34:34,969 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1694032470260_default_test-1694032296496 finished. 2023-09-06T20:34:34,969 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1014): test-1694032296496 2023-09-06T20:34:34,974 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1694032467935 2023-09-06T20:34:34,974 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-09-06T20:34:34,978 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:39327/backupUT/backup_1694032467935/.backup.manifest 2023-09-06T20:34:34,979 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(358): Backup backup_1694032467935 finished: type=FULL,tablelist=test-1694032296496,targetRootDir=hdfs://localhost:39327/backupUT,startts=1694032470070,completets=1694032474974,bytescopied=0 2023-09-06T20:34:34,979 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-09-06T20:34:34,979 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1694032470260_default_test-1694032296496 2023-09-06T20:34:34,979 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1694032470260_default_test-1694032296496" 2023-09-06T20:34:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1694032470260_default_test-1694032296496 2023-09-06T20:34:34,982 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1694032470260_default_test-1694032296496 for backup backup_1694032467935 succeeded. 2023-09-06T20:34:34,983 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:34:34,986 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:34:34,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:34:34,988 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:34:34,990 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:34:34,994 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(376): Backup backup_1694032467935 completed. 2023-09-06T20:34:34,995 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:34,995 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] backup.TestFullRestore(80): backup complete 2023-09-06T20:34:35,009 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36a27e40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:35,010 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:35,011 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39880, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:35,012 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(101): HBase table table1 does not exist. It will be created during restore process 2023-09-06T20:34:35,012 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:39327/backupUT/backup_1694032467935/.backup.manifest 2023-09-06T20:34:35,014 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032467935/.backup.manifest 2023-09-06T20:34:35,014 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:39327/backupUT/backup_1694032467935/.backup.manifest 2023-09-06T20:34:35,016 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032467935/.backup.manifest 2023-09-06T20:34:35,016 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(148): Restoring 'test-1694032296496' to 'table1' from full backup image hdfs://localhost:39327/backupUT/backup_1694032467935/default/test-1694032296496 2023-09-06T20:34:35,022 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(487): Creating target table 'table1' 2023-09-06T20:34:35,022 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:39327/backupUT/backup_1694032467935/default/test-1694032296496/archive/data/default/test-1694032296496/1f072a702e0732d4b296550713fb6262 2023-09-06T20:34:35,022 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:39327/backupUT/backup_1694032467935/default/test-1694032296496/archive/data/default/test-1694032296496/1f072a702e0732d4b296550713fb6262/f in region [hdfs://localhost:39327/backupUT/backup_1694032467935/default/test-1694032296496/archive/data/default/test-1694032296496/1f072a702e0732d4b296550713fb6262] 2023-09-06T20:34:35,025 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:39327/backupUT/backup_1694032467935/default/test-1694032296496/archive/data/default/test-1694032296496/1f072a702e0732d4b296550713fb6262/f/8c33b1e0971346619bc1707bbedeefce_SeqId_4_ first=row0 last=row98 2023-09-06T20:34:35,026 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:34:35,026 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48288, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:34:35,027 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:34:35,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=161, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=table1 2023-09-06T20:34:35,030 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:34:35,030 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:34:35,031 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 161 2023-09-06T20:34:35,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=161 2023-09-06T20:34:35,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=161 2023-09-06T20:34:35,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=161 2023-09-06T20:34:35,436 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => d1658bb1289fb3b665bc8b667a5849d5, NAME => 'table1,,1694032475027.d1658bb1289fb3b665bc8b667a5849d5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:34:35,442 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(888): Instantiated table1,,1694032475027.d1658bb1289fb3b665bc8b667a5849d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:34:35,442 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1668): Closing d1658bb1289fb3b665bc8b667a5849d5, disabling compactions & flushes 2023-09-06T20:34:35,442 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1690): Closing region table1,,1694032475027.d1658bb1289fb3b665bc8b667a5849d5. 2023-09-06T20:34:35,442 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1694032475027.d1658bb1289fb3b665bc8b667a5849d5. 2023-09-06T20:34:35,442 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on table1,,1694032475027.d1658bb1289fb3b665bc8b667a5849d5. after waiting 0 ms 2023-09-06T20:34:35,442 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region table1,,1694032475027.d1658bb1289fb3b665bc8b667a5849d5. 2023-09-06T20:34:35,442 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1909): Closed table1,,1694032475027.d1658bb1289fb3b665bc8b667a5849d5. 2023-09-06T20:34:35,442 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1622): Region close journal for d1658bb1289fb3b665bc8b667a5849d5: 2023-09-06T20:34:35,443 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:34:35,443 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"table1,,1694032475027.d1658bb1289fb3b665bc8b667a5849d5.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1694032475443"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032475443"}]},"ts":"1694032475443"} 2023-09-06T20:34:35,444 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:34:35,445 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:34:35,445 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032475445"}]},"ts":"1694032475445"} 2023-09-06T20:34:35,445 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLING in hbase:meta 2023-09-06T20:34:35,451 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=d1658bb1289fb3b665bc8b667a5849d5, ASSIGN}] 2023-09-06T20:34:35,453 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=d1658bb1289fb3b665bc8b667a5849d5, ASSIGN 2023-09-06T20:34:35,453 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=table1, region=d1658bb1289fb3b665bc8b667a5849d5, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36961,1694032287907; forceNewPlan=false, retain=false 2023-09-06T20:34:35,607 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=162 updating hbase:meta row=d1658bb1289fb3b665bc8b667a5849d5, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:35,610 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE; OpenRegionProcedure d1658bb1289fb3b665bc8b667a5849d5, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:34:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=161 2023-09-06T20:34:35,764 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] handler.AssignRegionHandler(130): Open table1,,1694032475027.d1658bb1289fb3b665bc8b667a5849d5. 2023-09-06T20:34:35,765 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(7254): Opening region: {ENCODED => d1658bb1289fb3b665bc8b667a5849d5, NAME => 'table1,,1694032475027.d1658bb1289fb3b665bc8b667a5849d5.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:34:35,765 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:34:35,765 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 d1658bb1289fb3b665bc8b667a5849d5 2023-09-06T20:34:35,765 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(888): Instantiated table1,,1694032475027.d1658bb1289fb3b665bc8b667a5849d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:34:35,765 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(7296): checking encryption for d1658bb1289fb3b665bc8b667a5849d5 2023-09-06T20:34:35,765 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(7299): checking classloading for d1658bb1289fb3b665bc8b667a5849d5 2023-09-06T20:34:35,766 INFO [StoreOpener-d1658bb1289fb3b665bc8b667a5849d5-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region d1658bb1289fb3b665bc8b667a5849d5 2023-09-06T20:34:35,767 INFO [StoreOpener-d1658bb1289fb3b665bc8b667a5849d5-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1658bb1289fb3b665bc8b667a5849d5 columnFamilyName f 2023-09-06T20:34:35,768 DEBUG [StoreOpener-d1658bb1289fb3b665bc8b667a5849d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:34:35,768 INFO [StoreOpener-d1658bb1289fb3b665bc8b667a5849d5-1 {}] regionserver.HStore(324): Store=d1658bb1289fb3b665bc8b667a5849d5/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:34:35,769 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/table1/d1658bb1289fb3b665bc8b667a5849d5 2023-09-06T20:34:35,769 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/table1/d1658bb1289fb3b665bc8b667a5849d5 2023-09-06T20:34:35,770 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(1076): writing seq id for d1658bb1289fb3b665bc8b667a5849d5 2023-09-06T20:34:35,772 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/table1/d1658bb1289fb3b665bc8b667a5849d5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:34:35,772 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(1093): Opened d1658bb1289fb3b665bc8b667a5849d5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9965003520, jitterRate=-0.0719367265701294}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:34:35,773 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(992): Region open journal for d1658bb1289fb3b665bc8b667a5849d5: 2023-09-06T20:34:35,774 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegionServer(2556): Post open deploy tasks for table1,,1694032475027.d1658bb1289fb3b665bc8b667a5849d5., pid=163, masterSystemTime=1694032475762 2023-09-06T20:34:35,775 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegionServer(2583): Finished post open deploy task for table1,,1694032475027.d1658bb1289fb3b665bc8b667a5849d5. 2023-09-06T20:34:35,775 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] handler.AssignRegionHandler(158): Opened table1,,1694032475027.d1658bb1289fb3b665bc8b667a5849d5. 2023-09-06T20:34:35,775 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=162 updating hbase:meta row=d1658bb1289fb3b665bc8b667a5849d5, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:35,778 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=163, resume processing ppid=162 2023-09-06T20:34:35,778 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=163, ppid=162, state=SUCCESS; OpenRegionProcedure d1658bb1289fb3b665bc8b667a5849d5, server=jenkins-hbase3.apache.org,36961,1694032287907 in 168 msec 2023-09-06T20:34:35,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=162, resume processing ppid=161 2023-09-06T20:34:35,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=162, ppid=161, state=SUCCESS; TransitRegionStateProcedure table=table1, region=d1658bb1289fb3b665bc8b667a5849d5, ASSIGN in 327 msec 2023-09-06T20:34:35,780 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:34:35,780 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032475780"}]},"ts":"1694032475780"} 2023-09-06T20:34:35,782 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLED in hbase:meta 2023-09-06T20:34:35,784 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:34:35,785 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=161, state=SUCCESS; CreateTableProcedure table=table1 in 757 msec 2023-09-06T20:34:36,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=161 2023-09-06T20:34:36,140 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:39327/backupUT/backup_1694032467935/default/test-1694032296496/archive/data/default/test-1694032296496/1f072a702e0732d4b296550713fb6262 from hbase tables test-1694032296496 to tables table1 2023-09-06T20:34:36,140 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1694032296496 into table1 2023-09-06T20:34:36,141 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-table1-1694032476140 from hdfs://localhost:39327/backupUT/backup_1694032467935/default/test-1694032296496/archive/data/default/test-1694032296496/1f072a702e0732d4b296550713fb6262 2023-09-06T20:34:36,147 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1606e713, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:36,148 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:34:36,149 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48292, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:34:36,150 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-09-06T20:34:36,150 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table table1 2023-09-06T20:34:36,153 DEBUG [hconnection-0x90e9e09-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:36,154 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39886, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:36,155 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for table1: 2023-09-06T20:34:36,155 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-09-06T20:34:36,155 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/partitions_bf2750f0-41b9-4844-ae64-6b55edf33d2b 2023-09-06T20:34:36,561 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:36,561 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:36,562 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:36,562 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:36,723 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2023-09-06T20:34:37,557 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop-5191504983474812146.jar 2023-09-06T20:34:37,558 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:37,558 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:37,558 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:37,559 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:37,559 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:37,559 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:37,560 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:37,560 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:37,560 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:37,561 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:34:37,562 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:34:37,562 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:34:37,562 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:34:37,563 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:34:37,563 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:34:37,563 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:34:37,564 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:34:37,564 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:34:37,564 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:34:37,565 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:34:37,565 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:34:37,566 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:37,566 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:37,566 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:37,567 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:37,567 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:37,567 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:37,568 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:34:37,568 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: table1 2023-09-06T20:34:37,568 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:37,568 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-09-06T20:34:37,568 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-09-06T20:34:37,569 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:34:37,580 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-09-06T20:34:37,655 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032477621/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:34:37,655 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032477621/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:34:37,725 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:39327/backupUT/backup_1694032467935/default/test-1694032296496/archive/data/default/test-1694032296496/1f072a702e0732d4b296550713fb6262/f/8c33b1e0971346619bc1707bbedeefce_SeqId_4_ 2023-09-06T20:34:37,728 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-09-06T20:34:37,753 DEBUG [pool-535-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dc1a61e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:37,754 DEBUG [hconnection-0x1e2c5ee6-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:37,756 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39898, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:37,759 DEBUG [pool-535-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:37,759 DEBUG [pool-535-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-09-06T20:34:37,759 DEBUG [pool-535-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-09-06T20:34:37,763 DEBUG [pool-535-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-table1-1694032476140/_temporary/0/_temporary/attempt_local1552641587_0025_r_000000_0/f/b29ac7345e3548b7aa60d69d6748e2d9 is 37, key is row10/f:q1/1694032299136/Put/seqid=0 2023-09-06T20:34:37,764 WARN [Thread-2349 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:36961] Specified favored nodes: [jenkins-hbase3.apache.org:36961] 2023-09-06T20:34:38,699 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-table1-1694032476140 2023-09-06T20:34:38,704 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@156efa4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:38,705 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:38,707 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39914, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:38,712 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-table1-1694032476140/_SUCCESS 2023-09-06T20:34:38,715 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:34:38,716 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48298, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:34:38,724 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-table1-1694032476140/f/b29ac7345e3548b7aa60d69d6748e2d9 first=Optional[row0] last=Optional[row98] 2023-09-06T20:34:38,725 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=table1,,1694032475027.d1658bb1289fb3b665bc8b667a5849d5., hostname=jenkins-hbase3.apache.org,36961,1694032287907, seqNum=2 for row with hfile group [{f,hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-table1-1694032476140/f/b29ac7345e3548b7aa60d69d6748e2d9}] 2023-09-06T20:34:38,729 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-table1-1694032476140/f/b29ac7345e3548b7aa60d69d6748e2d9 for inclusion in d1658bb1289fb3b665bc8b667a5849d5/f 2023-09-06T20:34:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-09-06T20:34:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(625): Region bounds: first= last= 2023-09-06T20:34:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HRegion(2520): Flush status journal for d1658bb1289fb3b665bc8b667a5849d5: 2023-09-06T20:34:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/hbase-staging/bulk_output-default-table1-1694032476140/f/b29ac7345e3548b7aa60d69d6748e2d9 to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__table1__6bjdj2fva044p1qdbf8ppqjnjjac7aqbio1qv8b3fah0tctr05qh4c113g8d5pcc/f/b29ac7345e3548b7aa60d69d6748e2d9 2023-09-06T20:34:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__table1__6bjdj2fva044p1qdbf8ppqjnjjac7aqbio1qv8b3fah0tctr05qh4c113g8d5pcc/f/b29ac7345e3548b7aa60d69d6748e2d9 as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/table1/d1658bb1289fb3b665bc8b667a5849d5/f/5449aad0898b464bad2a85133c75df3e_SeqId_4_ 2023-09-06T20:34:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a86a5bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:34:38,741 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48304, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:34:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:38,745 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39922, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:38,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:38,756 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__table1__6bjdj2fva044p1qdbf8ppqjnjjac7aqbio1qv8b3fah0tctr05qh4c113g8d5pcc/f/b29ac7345e3548b7aa60d69d6748e2d9 into d1658bb1289fb3b665bc8b667a5849d5/f as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/table1/d1658bb1289fb3b665bc8b667a5849d5/f/5449aad0898b464bad2a85133c75df3e_SeqId_4_ - updating store file list. 2023-09-06T20:34:38,760 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/table1/d1658bb1289fb3b665bc8b667a5849d5/f/5449aad0898b464bad2a85133c75df3e_SeqId_4_ into d1658bb1289fb3b665bc8b667a5849d5/f 2023-09-06T20:34:38,760 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__table1__6bjdj2fva044p1qdbf8ppqjnjjac7aqbio1qv8b3fah0tctr05qh4c113g8d5pcc/f/b29ac7345e3548b7aa60d69d6748e2d9 into d1658bb1289fb3b665bc8b667a5849d5/f (new location: hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/table1/d1658bb1289fb3b665bc8b667a5849d5/f/5449aad0898b464bad2a85133c75df3e_SeqId_4_) 2023-09-06T20:34:38,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/staging/jenkins__table1__6bjdj2fva044p1qdbf8ppqjnjjac7aqbio1qv8b3fah0tctr05qh4c113g8d5pcc/f/b29ac7345e3548b7aa60d69d6748e2d9 2023-09-06T20:34:38,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ebf9a79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:34:38,769 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48310, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-09-06T20:34:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:38,773 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39928, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-09-06T20:34:38,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36961 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:38,785 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:38,785 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-09-06T20:34:38,785 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-09-06T20:34:38,785 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(233): Backup: backup_1694032467935 hdfs://localhost:39327/backupUT/backup_1694032467935/default/test-1694032296496/ 2023-09-06T20:34:38,785 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-09-06T20:34:38,785 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:38,787 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable table1 2023-09-06T20:34:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=table1 2023-09-06T20:34:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=164 2023-09-06T20:34:38,790 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032478790"}]},"ts":"1694032478790"} 2023-09-06T20:34:38,791 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLING in hbase:meta 2023-09-06T20:34:38,794 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(293): Set table1 to state=DISABLING 2023-09-06T20:34:38,794 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=d1658bb1289fb3b665bc8b667a5849d5, UNASSIGN}] 2023-09-06T20:34:38,795 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=165, ppid=164, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=d1658bb1289fb3b665bc8b667a5849d5, UNASSIGN 2023-09-06T20:34:38,796 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=165 updating hbase:meta row=d1658bb1289fb3b665bc8b667a5849d5, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:38,797 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE; CloseRegionProcedure d1658bb1289fb3b665bc8b667a5849d5, server=jenkins-hbase3.apache.org,36961,1694032287907}] 2023-09-06T20:34:38,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=164 2023-09-06T20:34:38,948 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] handler.UnassignRegionHandler(119): Close d1658bb1289fb3b665bc8b667a5849d5 2023-09-06T20:34:38,949 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1668): Closing d1658bb1289fb3b665bc8b667a5849d5, disabling compactions & flushes 2023-09-06T20:34:38,949 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1690): Closing region table1,,1694032475027.d1658bb1289fb3b665bc8b667a5849d5. 2023-09-06T20:34:38,949 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1694032475027.d1658bb1289fb3b665bc8b667a5849d5. 2023-09-06T20:34:38,949 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1778): Acquired close lock on table1,,1694032475027.d1658bb1289fb3b665bc8b667a5849d5. after waiting 0 ms 2023-09-06T20:34:38,949 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1788): Updates disabled for region table1,,1694032475027.d1658bb1289fb3b665bc8b667a5849d5. 2023-09-06T20:34:38,955 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/table1/d1658bb1289fb3b665bc8b667a5849d5/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-09-06T20:34:38,955 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:34:38,955 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1909): Closed table1,,1694032475027.d1658bb1289fb3b665bc8b667a5849d5. 2023-09-06T20:34:38,956 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1622): Region close journal for d1658bb1289fb3b665bc8b667a5849d5: 2023-09-06T20:34:38,958 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] handler.UnassignRegionHandler(163): Closed d1658bb1289fb3b665bc8b667a5849d5 2023-09-06T20:34:38,958 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=165 updating hbase:meta row=d1658bb1289fb3b665bc8b667a5849d5, regionState=CLOSED 2023-09-06T20:34:38,960 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=166, resume processing ppid=165 2023-09-06T20:34:38,960 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=166, ppid=165, state=SUCCESS; CloseRegionProcedure d1658bb1289fb3b665bc8b667a5849d5, server=jenkins-hbase3.apache.org,36961,1694032287907 in 162 msec 2023-09-06T20:34:38,962 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=165, resume processing ppid=164 2023-09-06T20:34:38,962 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=165, ppid=164, state=SUCCESS; TransitRegionStateProcedure table=table1, region=d1658bb1289fb3b665bc8b667a5849d5, UNASSIGN in 166 msec 2023-09-06T20:34:38,962 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032478962"}]},"ts":"1694032478962"} 2023-09-06T20:34:38,963 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLED in hbase:meta 2023-09-06T20:34:38,964 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set table1 to state=DISABLED 2023-09-06T20:34:38,966 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=164, state=SUCCESS; DisableTableProcedure table=table1 in 178 msec 2023-09-06T20:34:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=164 2023-09-06T20:34:39,092 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster$5(2448): Client=jenkins//172.31.12.81 delete table1 2023-09-06T20:34:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=167, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=table1 2023-09-06T20:34:39,095 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=167, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-09-06T20:34:39,096 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=167, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=table1 2023-09-06T20:34:39,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=167 2023-09-06T20:34:39,098 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/table1/d1658bb1289fb3b665bc8b667a5849d5 2023-09-06T20:34:39,099 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/table1/d1658bb1289fb3b665bc8b667a5849d5/f, FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/table1/d1658bb1289fb3b665bc8b667a5849d5/recovered.edits] 2023-09-06T20:34:39,103 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/table1/d1658bb1289fb3b665bc8b667a5849d5/f/5449aad0898b464bad2a85133c75df3e_SeqId_4_ to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/archive/data/default/table1/d1658bb1289fb3b665bc8b667a5849d5/f/5449aad0898b464bad2a85133c75df3e_SeqId_4_ 2023-09-06T20:34:39,105 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/table1/d1658bb1289fb3b665bc8b667a5849d5/recovered.edits/6.seqid to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/archive/data/default/table1/d1658bb1289fb3b665bc8b667a5849d5/recovered.edits/6.seqid 2023-09-06T20:34:39,106 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/table1/d1658bb1289fb3b665bc8b667a5849d5 2023-09-06T20:34:39,106 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(293): Archived table1 regions 2023-09-06T20:34:39,108 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=167, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=table1 2023-09-06T20:34:39,109 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of table1 from hbase:meta 2023-09-06T20:34:39,111 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(387): Removing 'table1' descriptor. 2023-09-06T20:34:39,111 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=167, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=table1 2023-09-06T20:34:39,111 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(377): Removing 'table1' from region states. 2023-09-06T20:34:39,112 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1,,1694032475027.d1658bb1289fb3b665bc8b667a5849d5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1694032479111"}]},"ts":"9223372036854775807"} 2023-09-06T20:34:39,113 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-06T20:34:39,113 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => d1658bb1289fb3b665bc8b667a5849d5, NAME => 'table1,,1694032475027.d1658bb1289fb3b665bc8b667a5849d5.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:34:39,113 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(381): Marking 'table1' as deleted. 2023-09-06T20:34:39,113 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1694032479113"}]},"ts":"9223372036854775807"} 2023-09-06T20:34:39,115 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table table1 state from META 2023-09-06T20:34:39,116 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(127): Finished pid=167, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-09-06T20:34:39,117 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=167, state=SUCCESS; DeleteTableProcedure table=table1 in 24 msec 2023-09-06T20:34:39,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=167 2023-09-06T20:34:39,213 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingleCommand Thread=605 (was 603) - Thread LEAK? -, OpenFileDescriptor=692 (was 695), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=145 (was 161), ProcessCount=167 (was 167), AvailableMemoryMB=3181 (was 3182) 2023-09-06T20:34:39,213 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(130): Thread=605 is superior to 500 2023-09-06T20:34:39,228 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultipleDNE Thread=605, OpenFileDescriptor=692, MaxFileDescriptor=60000, SystemLoadAverage=145, ProcessCount=167, AvailableMemoryMB=3181 2023-09-06T20:34:39,228 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(130): Thread=605 is superior to 500 2023-09-06T20:34:39,228 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] backup.TestFullRestore(302): test restore fails on multiple tables that do not exist 2023-09-06T20:34:39,233 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d6cd93b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:39,236 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:39,238 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39944, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:39,240 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:34:39,241 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48318, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:34:39,250 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:34:39,254 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:34:39,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032479254 (current time:1694032479254). 2023-09-06T20:34:39,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:34:39,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:34:39,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a42ed1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:39,263 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39952, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:39,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:39,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:34:39,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=168, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:34:39,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 168 2023-09-06T20:34:39,267 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:34:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=168 2023-09-06T20:34:39,268 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:34:39,269 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:34:39,274 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:34:39,274 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281}] 2023-09-06T20:34:39,276 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=169, ppid=168, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 2023-09-06T20:34:39,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=168 2023-09-06T20:34:39,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=169 2023-09-06T20:34:39,427 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:34:39,427 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HRegion(2819): Flushing cf63a85c53279bc94811b50578f31281 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-09-06T20:34:39,431 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/ce4d996e86ba4032a6c69c75e37f90ab is 169, key is trslm:hdfs://localhost:39327/backupUT\x00test-1694032296496/meta:log-roll-map/1694032474971/Put/seqid=0 2023-09-06T20:34:39,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=168 2023-09-06T20:34:39,835 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/ce4d996e86ba4032a6c69c75e37f90ab 2023-09-06T20:34:39,842 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/812eea73f9bf4d6ebfc9ca4d4bee5ac3 is 452, key is session:backup_1694032467935/session:context/1694032474989/Put/seqid=0 2023-09-06T20:34:39,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=168 2023-09-06T20:34:40,246 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/812eea73f9bf4d6ebfc9ca4d4bee5ac3 2023-09-06T20:34:40,251 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/ce4d996e86ba4032a6c69c75e37f90ab as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/ce4d996e86ba4032a6c69c75e37f90ab 2023-09-06T20:34:40,256 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/ce4d996e86ba4032a6c69c75e37f90ab, entries=4, sequenceid=138, filesize=5.5 K 2023-09-06T20:34:40,256 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/812eea73f9bf4d6ebfc9ca4d4bee5ac3 as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/812eea73f9bf4d6ebfc9ca4d4bee5ac3 2023-09-06T20:34:40,260 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/812eea73f9bf4d6ebfc9ca4d4bee5ac3, entries=2, sequenceid=138, filesize=5.4 K 2023-09-06T20:34:40,261 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for cf63a85c53279bc94811b50578f31281 in 834ms, sequenceid=138, compaction requested=true 2023-09-06T20:34:40,261 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HRegion(2520): Flush status journal for cf63a85c53279bc94811b50578f31281: 2023-09-06T20:34:40,261 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. for snapshot_backup_system completed. 2023-09-06T20:34:40,261 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:34:40,261 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:34:40,261 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/0379b1e1f59e40d2ac2991b2ae6b86ce, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/dd7edbeade334fa7895ff079e0fc3472, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/78eec26cfa0242e48353b19a4a00b7d5, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/faffbcb72f8649bcad705936a3105fd4, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/01fb4a2eb68c4823807811190aa11f8b, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/b14bcdd382934779b0fa80ae042378aa, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/b7aca3aa47f24516bd2686bfd6b9193d, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/48f5b68b63f84de7a7c1f4f08993f64f, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/3ff5f0a110934ea19f13a9dc5a7d2318, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/5c1aa6dd12864ef4a9f61d102553701d, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/ba7080b60f13447ca7b0f338dd848e23, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/ce4d996e86ba4032a6c69c75e37f90ab] hfiles 2023-09-06T20:34:40,261 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (1/12): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/0379b1e1f59e40d2ac2991b2ae6b86ce for snapshot=snapshot_backup_system 2023-09-06T20:34:40,261 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (2/12): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/dd7edbeade334fa7895ff079e0fc3472 for snapshot=snapshot_backup_system 2023-09-06T20:34:40,261 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (3/12): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/78eec26cfa0242e48353b19a4a00b7d5 for snapshot=snapshot_backup_system 2023-09-06T20:34:40,261 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (4/12): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/faffbcb72f8649bcad705936a3105fd4 for snapshot=snapshot_backup_system 2023-09-06T20:34:40,261 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (5/12): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/01fb4a2eb68c4823807811190aa11f8b for snapshot=snapshot_backup_system 2023-09-06T20:34:40,261 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (6/12): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/b14bcdd382934779b0fa80ae042378aa for snapshot=snapshot_backup_system 2023-09-06T20:34:40,261 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (7/12): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/b7aca3aa47f24516bd2686bfd6b9193d for snapshot=snapshot_backup_system 2023-09-06T20:34:40,261 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (8/12): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/48f5b68b63f84de7a7c1f4f08993f64f for snapshot=snapshot_backup_system 2023-09-06T20:34:40,261 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (9/12): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/3ff5f0a110934ea19f13a9dc5a7d2318 for snapshot=snapshot_backup_system 2023-09-06T20:34:40,261 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (10/12): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/5c1aa6dd12864ef4a9f61d102553701d for snapshot=snapshot_backup_system 2023-09-06T20:34:40,261 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (11/12): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/ba7080b60f13447ca7b0f338dd848e23 for snapshot=snapshot_backup_system 2023-09-06T20:34:40,261 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (12/12): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/ce4d996e86ba4032a6c69c75e37f90ab for snapshot=snapshot_backup_system 2023-09-06T20:34:40,262 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/4ced064cdbd34bd6b594c76f049fa4ee, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/baca9a30045a4dabad9b3b78a8fd6c86, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/bcfde09dc46641bebf14d756f321d0af, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/2dc68dbd9f3749a1a2d9d7333fa4771d, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/6da826e7b47e41daa41e268fb4e951b2, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/849e36507ab34f85a508bffbc6e215d3, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/8bd33efd4a8e44d1b5b9f6a66a166990, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/13357027918440549abec030e3035318, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/a2912410595b4f2d8ed499b1de1206c9, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/c1202b8538934be1bed1ea11b0457324, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/df36c3270e704eb2966dff949ba7f613, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/060ea6a84a9a417fabf8ead571bb96d5, hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/812eea73f9bf4d6ebfc9ca4d4bee5ac3] hfiles 2023-09-06T20:34:40,262 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (1/13): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/4ced064cdbd34bd6b594c76f049fa4ee for snapshot=snapshot_backup_system 2023-09-06T20:34:40,262 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (2/13): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/baca9a30045a4dabad9b3b78a8fd6c86 for snapshot=snapshot_backup_system 2023-09-06T20:34:40,262 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (3/13): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/bcfde09dc46641bebf14d756f321d0af for snapshot=snapshot_backup_system 2023-09-06T20:34:40,262 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (4/13): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/2dc68dbd9f3749a1a2d9d7333fa4771d for snapshot=snapshot_backup_system 2023-09-06T20:34:40,262 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (5/13): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/6da826e7b47e41daa41e268fb4e951b2 for snapshot=snapshot_backup_system 2023-09-06T20:34:40,262 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (6/13): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/849e36507ab34f85a508bffbc6e215d3 for snapshot=snapshot_backup_system 2023-09-06T20:34:40,262 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (7/13): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/8bd33efd4a8e44d1b5b9f6a66a166990 for snapshot=snapshot_backup_system 2023-09-06T20:34:40,262 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (8/13): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/13357027918440549abec030e3035318 for snapshot=snapshot_backup_system 2023-09-06T20:34:40,262 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (9/13): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/a2912410595b4f2d8ed499b1de1206c9 for snapshot=snapshot_backup_system 2023-09-06T20:34:40,262 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (10/13): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/c1202b8538934be1bed1ea11b0457324 for snapshot=snapshot_backup_system 2023-09-06T20:34:40,262 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (11/13): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/df36c3270e704eb2966dff949ba7f613 for snapshot=snapshot_backup_system 2023-09-06T20:34:40,262 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (12/13): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/060ea6a84a9a417fabf8ead571bb96d5 for snapshot=snapshot_backup_system 2023-09-06T20:34:40,262 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (13/13): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/812eea73f9bf4d6ebfc9ca4d4bee5ac3 for snapshot=snapshot_backup_system 2023-09-06T20:34:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=168 2023-09-06T20:34:40,667 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:34:40,667 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=169 2023-09-06T20:34:40,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=169 2023-09-06T20:34:40,667 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region cf63a85c53279bc94811b50578f31281 2023-09-06T20:34:40,668 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=169, ppid=168, state=RUNNABLE; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 2023-09-06T20:34:40,670 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=169, resume processing ppid=168 2023-09-06T20:34:40,670 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:34:40,670 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=169, ppid=168, state=SUCCESS; SnapshotRegionProcedure cf63a85c53279bc94811b50578f31281 in 1.3950 sec 2023-09-06T20:34:40,671 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:34:40,671 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:34:40,671 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:34:40,672 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:34:41,081 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:34:41,096 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:34:41,096 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:34:41,097 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:34:41,097 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 168 2023-09-06T20:34:41,098 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=168, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8320 sec 2023-09-06T20:34:41,267 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:34:41,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=168 2023-09-06T20:34:41,372 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(115): Backup backup_1694032479233 started at 1694032481371. 2023-09-06T20:34:41,373 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(119): Backup session backup_1694032479233 has been started. 2023-09-06T20:34:41,377 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-09-06T20:34:41,378 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-09-06T20:34:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-09-06T20:34:41,378 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-09-06T20:34:41,378 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-09-06T20:34:41,379 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-09-06T20:34:41,380 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-09-06T20:34:41,380 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:34:41,382 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:41,382 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:34:41,382 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:34:41,382 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-09-06T20:34:41,382 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-09-06T20:34:41,383 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-09-06T20:34:41,383 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:34:41,383 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-09-06T20:34:41,383 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-09-06T20:34:41,383 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-09-06T20:34:41,383 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-09-06T20:34:41,383 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-09-06T20:34:41,384 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-09-06T20:34:41,384 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-09-06T20:34:41,384 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,36961,1694032287907' joining acquired barrier for procedure (rolllog) in zk 2023-09-06T20:34:41,386 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-09-06T20:34:41,386 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:41,386 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:34:41,386 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:34:41,387 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-09-06T20:34:41,387 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:34:41,387 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:34:41,387 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:41,388 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:41,388 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:34:41,388 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,36961,1694032287907' joining acquired barrier for procedure 'rolllog' on coordinator 2023-09-06T20:34:41,388 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@1ad7c4d4[Count = 0] remaining members to acquire global barrier 2023-09-06T20:34:41,388 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-09-06T20:34:41,388 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-09-06T20:34:41,390 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-09-06T20:34:41,390 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-09-06T20:34:41,390 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-09-06T20:34:41,390 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-09-06T20:34:41,390 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-09-06T20:34:41,390 DEBUG [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:41,390 INFO [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1694032470104 highest: 1694032470104 on jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:41,390 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta:.meta(num 1694032470088) roll requested 2023-09-06T20:34:41,408 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032481391.meta, exclude list is [], retry=0 2023-09-06T20:34:41,411 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK] 2023-09-06T20:34:41,413 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032470088.meta with entries=12, filesize=2.74 KB; new WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032481391.meta 2023-09-06T20:34:41,413 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK]] 2023-09-06T20:34:41,413 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032470088.meta is not closed yet, will try archiving it next time 2023-09-06T20:34:41,413 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36961%2C1694032287907:(num 1694032470104) roll requested 2023-09-06T20:34:41,415 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta.1694032470088.meta not finished, retry = 0 2023-09-06T20:34:41,427 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032481413, exclude list is [], retry=0 2023-09-06T20:34:41,430 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK] 2023-09-06T20:34:41,431 DEBUG [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 41 2023-09-06T20:34:41,431 INFO [rs(jenkins-hbase3.apache.org,36961,1694032287907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1694032481413 on jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:41,431 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032470104 with entries=13, filesize=3.13 KB; new WAL /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032481413 2023-09-06T20:34:41,431 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41603,DS-1d0182ba-524c-449c-b659-7241cc36c52e,DISK]] 2023-09-06T20:34:41,431 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032470104 is not closed yet, will try archiving it next time 2023-09-06T20:34:41,432 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032464047 to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/oldWALs/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032464047 2023-09-06T20:34:41,433 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032470104 not finished, retry = 0 2023-09-06T20:34:41,439 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-09-06T20:34:41,439 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,36961,1694032287907' in zk 2023-09-06T20:34:41,441 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-09-06T20:34:41,441 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:34:41,441 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:41,441 DEBUG [member: 'jenkins-hbase3.apache.org,36961,1694032287907' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-09-06T20:34:41,442 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:34:41,442 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:34:41,443 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:34:41,443 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:34:41,443 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:41,444 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:41,444 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:34:41,444 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:41,444 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:41,445 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,36961,1694032287907': 2023-09-06T20:34:41,445 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,36961,1694032287907' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-09-06T20:34:41,445 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-09-06T20:34:41,445 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-09-06T20:34:41,445 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-09-06T20:34:41,445 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-09-06T20:34:41,445 INFO [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-09-06T20:34:41,447 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-09-06T20:34:41,447 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:34:41,447 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:34:41,447 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-09-06T20:34:41,447 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:34:41,447 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:34:41,447 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:34:41,447 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:34:41,448 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:34:41,448 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:41,448 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:34:41,448 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:41,449 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:41,449 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:34:41,449 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:41,450 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:41,453 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:34:41,453 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:34:41,453 DEBUG [(jenkins-hbase3.apache.org,37419,1694032284546)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:34:41,453 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-09-06T20:34:41,453 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-09-06T20:34:41,454 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:34:41,454 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:34:41,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-09-06T20:34:41,557 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1694032481557_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } 2023-09-06T20:34:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032481557 (current time:1694032481557). 2023-09-06T20:34:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:34:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1694032481557_ns2_test-16940322964961 VERSION not specified, setting to 2 2023-09-06T20:34:41,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ffa3924, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:41,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:41,569 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51610, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:41,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:41,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:34:41,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=170, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1694032481557_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } 2023-09-06T20:34:41,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1694032481557_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 }, snapshot procedure id = 170 2023-09-06T20:34:41,573 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1694032481557_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:34:41,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=170 2023-09-06T20:34:41,573 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1694032481557_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:34:41,575 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1694032481557_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:34:41,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=170 2023-09-06T20:34:41,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=170 2023-09-06T20:34:41,982 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1694032481557_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:34:41,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE; SnapshotRegionProcedure e70b91c8ad9e774e6ad3f358efadcaf1}] 2023-09-06T20:34:41,983 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=171, ppid=170, state=RUNNABLE; SnapshotRegionProcedure e70b91c8ad9e774e6ad3f358efadcaf1 2023-09-06T20:34:42,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=171 2023-09-06T20:34:42,135 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16940322964961,,1694032434553.e70b91c8ad9e774e6ad3f358efadcaf1. 2023-09-06T20:34:42,135 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] regionserver.HRegion(2520): Flush status journal for e70b91c8ad9e774e6ad3f358efadcaf1: 2023-09-06T20:34:42,135 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16940322964961,,1694032434553.e70b91c8ad9e774e6ad3f358efadcaf1. for snapshot_1694032481557_ns2_test-16940322964961 completed. 2023-09-06T20:34:42,135 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16940322964961,,1694032434553.e70b91c8ad9e774e6ad3f358efadcaf1.' region-info for snapshot=snapshot_1694032481557_ns2_test-16940322964961 2023-09-06T20:34:42,135 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:34:42,135 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/e70b91c8ad9e774e6ad3f358efadcaf1/f/033098d0c12f4e7492b697713877c54c_SeqId_4_] hfiles 2023-09-06T20:34:42,135 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/e70b91c8ad9e774e6ad3f358efadcaf1/f/033098d0c12f4e7492b697713877c54c_SeqId_4_ for snapshot=snapshot_1694032481557_ns2_test-16940322964961 2023-09-06T20:34:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=170 2023-09-06T20:34:42,541 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16940322964961,,1694032434553.e70b91c8ad9e774e6ad3f358efadcaf1. 2023-09-06T20:34:42,541 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=171 2023-09-06T20:34:42,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=171 2023-09-06T20:34:42,541 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1694032481557_ns2_test-16940322964961 on region e70b91c8ad9e774e6ad3f358efadcaf1 2023-09-06T20:34:42,542 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=171, ppid=170, state=RUNNABLE; SnapshotRegionProcedure e70b91c8ad9e774e6ad3f358efadcaf1 2023-09-06T20:34:42,544 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=171, resume processing ppid=170 2023-09-06T20:34:42,544 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1694032481557_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:34:42,544 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=171, ppid=170, state=SUCCESS; SnapshotRegionProcedure e70b91c8ad9e774e6ad3f358efadcaf1 in 561 msec 2023-09-06T20:34:42,545 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1694032481557_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:34:42,545 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1694032481557_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:34:42,545 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1694032481557_ns2_test-16940322964961 2023-09-06T20:34:42,546 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032481557_ns2_test-16940322964961 2023-09-06T20:34:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=170 2023-09-06T20:34:42,957 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1694032481557_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:34:42,961 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1694032481557_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:34:42,962 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032481557_ns2_test-16940322964961 to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032481557_ns2_test-16940322964961 2023-09-06T20:34:42,963 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1694032481557_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:34:42,963 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1694032481557_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 }, snapshot procedure id = 170 2023-09-06T20:34:42,964 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=170, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1694032481557_ns2_test-16940322964961 table=ns2:test-16940322964961 type=FLUSH ttl=0 } in 1.3920 sec 2023-09-06T20:34:43,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=170 2023-09-06T20:34:43,679 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1694032483678_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } 2023-09-06T20:34:43,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032483679 (current time:1694032483679). 2023-09-06T20:34:43,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:34:43,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1694032483678_ns3_test-16940322964962 VERSION not specified, setting to 2 2023-09-06T20:34:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22c1cd30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:43,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:43,688 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51616, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:34:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure2.ProcedureExecutor(1032): Stored pid=172, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1694032483678_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } 2023-09-06T20:34:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1694032483678_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 }, snapshot procedure id = 172 2023-09-06T20:34:43,692 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1694032483678_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:34:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=172 2023-09-06T20:34:43,692 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1694032483678_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:34:43,694 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1694032483678_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:34:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=172 2023-09-06T20:34:43,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=172 2023-09-06T20:34:44,099 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1694032483678_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:34:44,100 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE; SnapshotRegionProcedure 895b7c16371d966da7eb221de358652c}] 2023-09-06T20:34:44,101 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=173, ppid=172, state=RUNNABLE; SnapshotRegionProcedure 895b7c16371d966da7eb221de358652c 2023-09-06T20:34:44,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36961 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=173 2023-09-06T20:34:44,253 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16940322964962,,1694032438327.895b7c16371d966da7eb221de358652c. 2023-09-06T20:34:44,253 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] regionserver.HRegion(2520): Flush status journal for 895b7c16371d966da7eb221de358652c: 2023-09-06T20:34:44,253 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16940322964962,,1694032438327.895b7c16371d966da7eb221de358652c. for snapshot_1694032483678_ns3_test-16940322964962 completed. 2023-09-06T20:34:44,253 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16940322964962,,1694032438327.895b7c16371d966da7eb221de358652c.' region-info for snapshot=snapshot_1694032483678_ns3_test-16940322964962 2023-09-06T20:34:44,253 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:34:44,253 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-09-06T20:34:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=172 2023-09-06T20:34:44,659 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16940322964962,,1694032438327.895b7c16371d966da7eb221de358652c. 2023-09-06T20:34:44,659 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=173 2023-09-06T20:34:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.HMaster(3995): Remote procedure done, pid=173 2023-09-06T20:34:44,659 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1694032483678_ns3_test-16940322964962 on region 895b7c16371d966da7eb221de358652c 2023-09-06T20:34:44,660 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=173, ppid=172, state=RUNNABLE; SnapshotRegionProcedure 895b7c16371d966da7eb221de358652c 2023-09-06T20:34:44,662 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=173, resume processing ppid=172 2023-09-06T20:34:44,662 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=173, ppid=172, state=SUCCESS; SnapshotRegionProcedure 895b7c16371d966da7eb221de358652c in 561 msec 2023-09-06T20:34:44,662 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1694032483678_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:34:44,663 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1694032483678_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:34:44,663 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1694032483678_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:34:44,663 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1694032483678_ns3_test-16940322964962 2023-09-06T20:34:44,664 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032483678_ns3_test-16940322964962 2023-09-06T20:34:44,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=172 2023-09-06T20:34:45,072 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1694032483678_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:34:45,077 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1694032483678_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:34:45,077 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/.tmp/snapshot_1694032483678_ns3_test-16940322964962 to hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032483678_ns3_test-16940322964962 2023-09-06T20:34:45,078 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1694032483678_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:34:45,078 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1694032483678_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 }, snapshot procedure id = 172 2023-09-06T20:34:45,079 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=172, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1694032483678_ns3_test-16940322964962 table=ns3:test-16940322964962 type=FLUSH ttl=0 } in 1.3890 sec 2023-09-06T20:34:45,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=172 2023-09-06T20:34:45,797 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1694032479233 2023-09-06T20:34:45,797 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-09-06T20:34:45,797 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-09-06T20:34:45,797 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1694032479233_ns3:test-16940322964962 2023-09-06T20:34:45,797 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1694032483678_ns3_test-16940322964962 to hdfs://localhost:39327/backupUT/backup_1694032479233/ns3/test-16940322964962/ 2023-09-06T20:34:45,797 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-09-06T20:34:45,819 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:34:45,820 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37419,1694032284546_-634948132_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:39327/backupUT/backup_1694032479233/ns3/test-16940322964962, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:39327/backupUT/backup_1694032479233/ns3/test-16940322964962/.hbase-snapshot/.tmp/snapshot_1694032483678_ns3_test-16940322964962 2023-09-06T20:34:45,820 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1, snapshotDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032483678_ns3_test-16940322964962. 2023-09-06T20:34:45,824 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032483678_ns3_test-16940322964962 to hdfs://localhost:39327/backupUT/backup_1694032479233/ns3/test-16940322964962/.hbase-snapshot/.tmp/snapshot_1694032483678_ns3_test-16940322964962 2023-09-06T20:34:46,238 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:46,238 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:46,239 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:46,239 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:47,217 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop-6531766826188674818.jar 2023-09-06T20:34:47,217 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:47,218 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:47,218 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:47,219 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:47,219 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:47,220 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:47,220 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:47,220 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:47,221 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:47,222 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:34:47,223 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:34:47,223 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:34:47,223 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:34:47,224 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:34:47,224 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:34:47,224 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:34:47,225 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:34:47,225 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:34:47,225 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:34:47,226 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:34:47,226 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:34:47,227 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:47,227 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:47,228 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:34:47,228 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:47,229 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:47,229 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:34:47,230 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:34:47,276 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1694032483678_ns3_test-16940322964962' hfile list 2023-09-06T20:34:47,359 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032487324/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:34:47,359 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032487324/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:34:48,388 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-09-06T20:34:48,389 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-09-06T20:34:48,394 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1694032483678_ns3_test-16940322964962 2023-09-06T20:34:48,395 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1694032483678_ns3_test-16940322964962 finished. 2023-09-06T20:34:48,395 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1694032479233_ns2:test-16940322964961 2023-09-06T20:34:48,395 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1694032481557_ns2_test-16940322964961 to hdfs://localhost:39327/backupUT/backup_1694032479233/ns2/test-16940322964961/ 2023-09-06T20:34:48,395 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-09-06T20:34:48,417 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1 2023-09-06T20:34:48,417 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37419,1694032284546_-129264002_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:39327/backupUT/backup_1694032479233/ns2/test-16940322964961, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:39327/backupUT/backup_1694032479233/ns2/test-16940322964961/.hbase-snapshot/.tmp/snapshot_1694032481557_ns2_test-16940322964961 2023-09-06T20:34:48,417 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:39327, inputRoot=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1, snapshotDir=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032481557_ns2_test-16940322964961. 2023-09-06T20:34:48,424 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/.hbase-snapshot/snapshot_1694032481557_ns2_test-16940322964961 to hdfs://localhost:39327/backupUT/backup_1694032479233/ns2/test-16940322964961/.hbase-snapshot/.tmp/snapshot_1694032481557_ns2_test-16940322964961 2023-09-06T20:34:48,840 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:48,841 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:48,841 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:48,842 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:49,807 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop-8474001524179182983.jar 2023-09-06T20:34:49,808 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:49,808 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:49,809 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:49,809 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:49,809 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:49,809 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:49,810 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:49,810 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:49,810 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:49,811 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:34:49,812 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:34:49,812 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:34:49,812 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:34:49,813 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:34:49,813 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:34:49,814 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:34:49,814 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:34:49,814 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:34:49,815 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:34:49,815 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:34:49,815 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:34:49,816 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:49,816 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:49,817 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:34:49,817 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:49,817 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:49,818 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:34:49,818 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:34:49,859 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1694032481557_ns2_test-16940322964961' hfile list 2023-09-06T20:34:49,861 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-09-06T20:34:49,930 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032489896/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:34:49,930 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/mapreduce.cluster.local.dir/1694032489896/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:34:49,984 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-09-06T20:34:50,007 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/e70b91c8ad9e774e6ad3f358efadcaf1/f/033098d0c12f4e7492b697713877c54c_SeqId_4_ output=hdfs://localhost:39327/backupUT/backup_1694032479233/ns2/test-16940322964961/archive/data/ns2/test-16940322964961/e70b91c8ad9e774e6ad3f358efadcaf1/f/033098d0c12f4e7492b697713877c54c_SeqId_4_ 2023-09-06T20:34:50,007 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8725 (8.5 K) time=0sec 8.321M/sec 2023-09-06T20:34:50,961 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-09-06T20:34:50,961 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-09-06T20:34:50,967 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1694032481557_ns2_test-16940322964961 2023-09-06T20:34:50,967 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1694032481557_ns2_test-16940322964961 finished. 2023-09-06T20:34:50,967 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1014): ns3:test-16940322964962 2023-09-06T20:34:50,967 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1014): ns2:test-16940322964961 2023-09-06T20:34:50,972 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1694032479233 2023-09-06T20:34:50,973 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-09-06T20:34:51,377 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:39327/backupUT/backup_1694032479233/.backup.manifest 2023-09-06T20:34:51,377 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1694032479233 2023-09-06T20:34:51,377 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-09-06T20:34:51,783 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:39327/backupUT/backup_1694032479233/.backup.manifest 2023-09-06T20:34:51,783 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(358): Backup backup_1694032479233 finished: type=FULL,tablelist=ns3:test-16940322964962;ns2:test-16940322964961,targetRootDir=hdfs://localhost:39327/backupUT,startts=1694032481371,completets=1694032490972,bytescopied=0 2023-09-06T20:34:51,783 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-09-06T20:34:51,783 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1694032483678_ns3_test-16940322964962 2023-09-06T20:34:51,784 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1694032483678_ns3_test-16940322964962" 2023-09-06T20:34:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1694032483678_ns3_test-16940322964962 2023-09-06T20:34:51,787 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1694032483678_ns3_test-16940322964962 for backup backup_1694032479233 succeeded. 2023-09-06T20:34:51,787 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1694032481557_ns2_test-16940322964961 2023-09-06T20:34:51,787 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1694032481557_ns2_test-16940322964961" 2023-09-06T20:34:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1694032481557_ns2_test-16940322964961 2023-09-06T20:34:51,790 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1694032481557_ns2_test-16940322964961 for backup backup_1694032479233 succeeded. 2023-09-06T20:34:51,790 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:34:51,794 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:34:51,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37419 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:34:51,797 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:34:51,798 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:34:51,803 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.TableBackupClient(376): Backup backup_1694032479233 completed. 2023-09-06T20:34:51,803 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:51,812 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(101): HBase table ns2:table2 does not exist. It will be created during restore process 2023-09-06T20:34:51,812 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(101): HBase table ns3:table3_restore does not exist. It will be created during restore process 2023-09-06T20:34:51,813 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:39327/backupUT/backup_1694032479233/.backup.manifest 2023-09-06T20:34:51,815 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032479233/.backup.manifest 2023-09-06T20:34:51,815 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:39327/backupUT/backup_1694032479233/.backup.manifest 2023-09-06T20:34:51,816 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032479233/.backup.manifest 2023-09-06T20:34:51,817 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:39327/backupUT/backup_1694032479233/.backup.manifest 2023-09-06T20:34:51,818 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1694032479233/.backup.manifest 2023-09-06T20:34:51,818 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] impl.RestoreTablesClient(148): Restoring 'faketable1' to 'ns2:table2' from full backup image hdfs://localhost:39327/backupUT/backup_1694032479233/default/faketable1 2023-09-06T20:34:51,837 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultipleDNE Thread=603 (was 605), OpenFileDescriptor=694 (was 692) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=141 (was 145), ProcessCount=167 (was 167), AvailableMemoryMB=3108 (was 3181) 2023-09-06T20:34:51,837 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.ResourceChecker(130): Thread=603 is superior to 500 2023-09-06T20:34:51,843 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-09-06T20:34:51,843 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:51,843 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-09-06T20:34:51,844 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] util.JVMClusterUtil(257): Found active master hash=1918682181, stopped=false 2023-09-06T20:34:51,844 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] master.ServerManager(910): Cluster shutdown requested of master=jenkins-hbase3.apache.org,37419,1694032284546 2023-09-06T20:34:51,846 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] procedure2.ProcedureExecutor(630): Stopping 2023-09-06T20:34:51,846 DEBUG [Listener at jenkins-hbase3.apache.org/33095 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:51,847 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,36961,1694032287907' ***** 2023-09-06T20:34:51,847 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] regionserver.HRegionServer(2531): STOPPED: Shutdown requested 2023-09-06T20:34:51,847 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-09-06T20:34:51,860 INFO [RS:0;jenkins-hbase3:36961 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@fbc47a8{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-09-06T20:34:51,865 INFO [RS:0;jenkins-hbase3:36961 {}] server.AbstractConnector(383): Stopped ServerConnector@324c5f54{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-09-06T20:34:51,865 INFO [RS:0;jenkins-hbase3:36961 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-09-06T20:34:51,866 INFO [RS:0;jenkins-hbase3:36961 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@3a910b99{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-09-06T20:34:51,866 INFO [RS:0;jenkins-hbase3:36961 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@63388c8d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop.log.dir/,STOPPED} 2023-09-06T20:34:51,869 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.HeapMemoryManager(220): Stopping 2023-09-06T20:34:51,869 INFO [RS:0;jenkins-hbase3:36961 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2023-09-06T20:34:51,869 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-09-06T20:34:51,869 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-09-06T20:34:51,869 INFO [RS:0;jenkins-hbase3:36961 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-09-06T20:34:51,869 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(3528): Received CLOSE for 895b7c16371d966da7eb221de358652c 2023-09-06T20:34:51,869 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(3528): Received CLOSE for 534ff531d20a23c3227d44d6f5ad3251 2023-09-06T20:34:51,869 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(3528): Received CLOSE for 1f072a702e0732d4b296550713fb6262 2023-09-06T20:34:51,869 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(3528): Received CLOSE for e70b91c8ad9e774e6ad3f358efadcaf1 2023-09-06T20:34:51,869 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(3528): Received CLOSE for 72ae4de93b6a56d1410c6ff0eb3e6243 2023-09-06T20:34:51,869 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(3528): Received CLOSE for cf63a85c53279bc94811b50578f31281 2023-09-06T20:34:51,869 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(3528): Received CLOSE for 3a1cfaf5b9def8649351000383a2d6de 2023-09-06T20:34:51,870 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,36961,1694032287907 2023-09-06T20:34:51,870 DEBUG [RS:0;jenkins-hbase3:36961 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:51,870 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-09-06T20:34:51,870 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-09-06T20:34:51,870 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-09-06T20:34:51,870 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(3528): Received CLOSE for 1588230740 2023-09-06T20:34:51,870 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 895b7c16371d966da7eb221de358652c, disabling compactions & flushes 2023-09-06T20:34:51,870 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-16940322964962,,1694032438327.895b7c16371d966da7eb221de358652c. 2023-09-06T20:34:51,870 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16940322964962,,1694032438327.895b7c16371d966da7eb221de358652c. 2023-09-06T20:34:51,870 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16940322964962,,1694032438327.895b7c16371d966da7eb221de358652c. after waiting 0 ms 2023-09-06T20:34:51,870 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16940322964962,,1694032438327.895b7c16371d966da7eb221de358652c. 2023-09-06T20:34:51,870 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(1579): Waiting on 8 regions to close 2023-09-06T20:34:51,870 DEBUG [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(1583): Online Regions={895b7c16371d966da7eb221de358652c=ns3:test-16940322964962,,1694032438327.895b7c16371d966da7eb221de358652c., 534ff531d20a23c3227d44d6f5ad3251=ns4:test-16940322964963,,1694032302569.534ff531d20a23c3227d44d6f5ad3251., 1f072a702e0732d4b296550713fb6262=test-1694032296496,,1694032379919.1f072a702e0732d4b296550713fb6262., e70b91c8ad9e774e6ad3f358efadcaf1=ns2:test-16940322964961,,1694032434553.e70b91c8ad9e774e6ad3f358efadcaf1., 72ae4de93b6a56d1410c6ff0eb3e6243=hbase:namespace,,1694032290929.72ae4de93b6a56d1410c6ff0eb3e6243., cf63a85c53279bc94811b50578f31281=backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281., 1588230740=hbase:meta,,1.1588230740, 3a1cfaf5b9def8649351000383a2d6de=backup:system_bulk,,1694032305079.3a1cfaf5b9def8649351000383a2d6de.} 2023-09-06T20:34:51,873 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-09-06T20:34:51,873 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-09-06T20:34:51,873 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-09-06T20:34:51,873 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-09-06T20:34:51,873 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-09-06T20:34:51,873 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=38.00 KB heapSize=66.30 KB 2023-09-06T20:34:51,873 DEBUG [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 1f072a702e0732d4b296550713fb6262, 3a1cfaf5b9def8649351000383a2d6de, 534ff531d20a23c3227d44d6f5ad3251, 72ae4de93b6a56d1410c6ff0eb3e6243, 895b7c16371d966da7eb221de358652c, cf63a85c53279bc94811b50578f31281, e70b91c8ad9e774e6ad3f358efadcaf1 2023-09-06T20:34:51,877 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns3/test-16940322964962/895b7c16371d966da7eb221de358652c/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-09-06T20:34:51,877 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:34:51,878 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-16940322964962,,1694032438327.895b7c16371d966da7eb221de358652c. 2023-09-06T20:34:51,878 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 895b7c16371d966da7eb221de358652c: 2023-09-06T20:34:51,878 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-16940322964962,,1694032438327.895b7c16371d966da7eb221de358652c. 2023-09-06T20:34:51,879 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 534ff531d20a23c3227d44d6f5ad3251, disabling compactions & flushes 2023-09-06T20:34:51,879 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-16940322964963,,1694032302569.534ff531d20a23c3227d44d6f5ad3251. 2023-09-06T20:34:51,879 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16940322964963,,1694032302569.534ff531d20a23c3227d44d6f5ad3251. 2023-09-06T20:34:51,879 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16940322964963,,1694032302569.534ff531d20a23c3227d44d6f5ad3251. after waiting 0 ms 2023-09-06T20:34:51,879 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16940322964963,,1694032302569.534ff531d20a23c3227d44d6f5ad3251. 2023-09-06T20:34:51,882 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns4/test-16940322964963/534ff531d20a23c3227d44d6f5ad3251/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-09-06T20:34:51,883 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:34:51,883 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-16940322964963,,1694032302569.534ff531d20a23c3227d44d6f5ad3251. 2023-09-06T20:34:51,883 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 534ff531d20a23c3227d44d6f5ad3251: 2023-09-06T20:34:51,884 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-16940322964963,,1694032302569.534ff531d20a23c3227d44d6f5ad3251. 2023-09-06T20:34:51,884 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 1f072a702e0732d4b296550713fb6262, disabling compactions & flushes 2023-09-06T20:34:51,884 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1694032296496,,1694032379919.1f072a702e0732d4b296550713fb6262. 2023-09-06T20:34:51,884 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1694032296496,,1694032379919.1f072a702e0732d4b296550713fb6262. 2023-09-06T20:34:51,884 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1694032296496,,1694032379919.1f072a702e0732d4b296550713fb6262. after waiting 0 ms 2023-09-06T20:34:51,884 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1694032296496,,1694032379919.1f072a702e0732d4b296550713fb6262. 2023-09-06T20:34:51,892 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/default/test-1694032296496/1f072a702e0732d4b296550713fb6262/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2023-09-06T20:34:51,893 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:34:51,893 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1694032296496,,1694032379919.1f072a702e0732d4b296550713fb6262. 2023-09-06T20:34:51,893 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 1f072a702e0732d4b296550713fb6262: 2023-09-06T20:34:51,894 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1694032296496,,1694032379919.1f072a702e0732d4b296550713fb6262. 2023-09-06T20:34:51,894 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing e70b91c8ad9e774e6ad3f358efadcaf1, disabling compactions & flushes 2023-09-06T20:34:51,894 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-16940322964961,,1694032434553.e70b91c8ad9e774e6ad3f358efadcaf1. 2023-09-06T20:34:51,894 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16940322964961,,1694032434553.e70b91c8ad9e774e6ad3f358efadcaf1. 2023-09-06T20:34:51,894 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16940322964961,,1694032434553.e70b91c8ad9e774e6ad3f358efadcaf1. after waiting 0 ms 2023-09-06T20:34:51,894 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16940322964961,,1694032434553.e70b91c8ad9e774e6ad3f358efadcaf1. 2023-09-06T20:34:51,899 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/ns2/test-16940322964961/e70b91c8ad9e774e6ad3f358efadcaf1/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2023-09-06T20:34:51,899 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:34:51,900 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-16940322964961,,1694032434553.e70b91c8ad9e774e6ad3f358efadcaf1. 2023-09-06T20:34:51,900 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for e70b91c8ad9e774e6ad3f358efadcaf1: 2023-09-06T20:34:51,900 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-16940322964961,,1694032434553.e70b91c8ad9e774e6ad3f358efadcaf1. 2023-09-06T20:34:51,901 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 72ae4de93b6a56d1410c6ff0eb3e6243, disabling compactions & flushes 2023-09-06T20:34:51,901 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1694032290929.72ae4de93b6a56d1410c6ff0eb3e6243. 2023-09-06T20:34:51,901 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1694032290929.72ae4de93b6a56d1410c6ff0eb3e6243. 2023-09-06T20:34:51,901 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1694032290929.72ae4de93b6a56d1410c6ff0eb3e6243. after waiting 0 ms 2023-09-06T20:34:51,901 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1694032290929.72ae4de93b6a56d1410c6ff0eb3e6243. 2023-09-06T20:34:51,901 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 72ae4de93b6a56d1410c6ff0eb3e6243 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-09-06T20:34:51,904 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/hbase/meta/1588230740/.tmp/info/5ed2dfe0ecf0490587f25fdffce75699 is 159, key is ns2:test-16940322964961,,1694032434553.e70b91c8ad9e774e6ad3f358efadcaf1./info:regioninfo/1694032435312/Put/seqid=0 2023-09-06T20:34:51,911 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.30 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/hbase/meta/1588230740/.tmp/info/5ed2dfe0ecf0490587f25fdffce75699 2023-09-06T20:34:51,916 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/hbase/namespace/72ae4de93b6a56d1410c6ff0eb3e6243/.tmp/info/ef4909236bc14d04837813a06ed4095d is 45, key is default/info:d/1694032292307/Put/seqid=0 2023-09-06T20:34:51,919 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-09-06T20:34:51,919 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-09-06T20:34:51,924 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-09-06T20:34:51,929 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/hbase/meta/1588230740/.tmp/rep_barrier/0627b1b20df14bae9a2900547c88dd87 is 107, key is ns2:test-16940322964961,,1694032299223.1932330ab1e654ad2ac9ae28fd5fcdb3./rep_barrier:/1694032318772/DeleteFamily/seqid=0 2023-09-06T20:34:51,933 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.13 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/hbase/meta/1588230740/.tmp/rep_barrier/0627b1b20df14bae9a2900547c88dd87 2023-09-06T20:34:51,950 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/hbase/meta/1588230740/.tmp/table/a2aebfbff2e7471d8e7316265ed073f9 is 101, key is ns2:test-16940322964961,,1694032299223.1932330ab1e654ad2ac9ae28fd5fcdb3./table:/1694032318772/DeleteFamily/seqid=0 2023-09-06T20:34:52,074 DEBUG [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 3a1cfaf5b9def8649351000383a2d6de, 72ae4de93b6a56d1410c6ff0eb3e6243, cf63a85c53279bc94811b50578f31281 2023-09-06T20:34:52,274 DEBUG [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 3a1cfaf5b9def8649351000383a2d6de, 72ae4de93b6a56d1410c6ff0eb3e6243, cf63a85c53279bc94811b50578f31281 2023-09-06T20:34:52,321 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/hbase/namespace/72ae4de93b6a56d1410c6ff0eb3e6243/.tmp/info/ef4909236bc14d04837813a06ed4095d 2023-09-06T20:34:52,326 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/hbase/namespace/72ae4de93b6a56d1410c6ff0eb3e6243/.tmp/info/ef4909236bc14d04837813a06ed4095d as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/hbase/namespace/72ae4de93b6a56d1410c6ff0eb3e6243/info/ef4909236bc14d04837813a06ed4095d 2023-09-06T20:34:52,330 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/hbase/namespace/72ae4de93b6a56d1410c6ff0eb3e6243/info/ef4909236bc14d04837813a06ed4095d, entries=7, sequenceid=11, filesize=5.1 K 2023-09-06T20:34:52,331 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 72ae4de93b6a56d1410c6ff0eb3e6243 in 430ms, sequenceid=11, compaction requested=false 2023-09-06T20:34:52,337 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/hbase/namespace/72ae4de93b6a56d1410c6ff0eb3e6243/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-09-06T20:34:52,337 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:34:52,338 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1694032290929.72ae4de93b6a56d1410c6ff0eb3e6243. 2023-09-06T20:34:52,338 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 72ae4de93b6a56d1410c6ff0eb3e6243: 2023-09-06T20:34:52,338 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1694032290929.72ae4de93b6a56d1410c6ff0eb3e6243. 2023-09-06T20:34:52,338 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing cf63a85c53279bc94811b50578f31281, disabling compactions & flushes 2023-09-06T20:34:52,338 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:34:52,338 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:34:52,338 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. after waiting 0 ms 2023-09-06T20:34:52,338 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:34:52,338 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing cf63a85c53279bc94811b50578f31281 2/2 column families, dataSize=1.46 KB heapSize=2.61 KB 2023-09-06T20:34:52,341 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/4f4da85a052d4e598b288ef7a835be8d is 171, key is trslm:hdfs://localhost:39327/backupUT\x00ns2:test-16940322964961/meta:log-roll-map/1694032490969/Put/seqid=0 2023-09-06T20:34:52,354 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.56 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/hbase/meta/1588230740/.tmp/table/a2aebfbff2e7471d8e7316265ed073f9 2023-09-06T20:34:52,358 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/hbase/meta/1588230740/.tmp/info/5ed2dfe0ecf0490587f25fdffce75699 as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/hbase/meta/1588230740/info/5ed2dfe0ecf0490587f25fdffce75699 2023-09-06T20:34:52,363 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/hbase/meta/1588230740/info/5ed2dfe0ecf0490587f25fdffce75699, entries=94, sequenceid=183, filesize=16.4 K 2023-09-06T20:34:52,364 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/hbase/meta/1588230740/.tmp/rep_barrier/0627b1b20df14bae9a2900547c88dd87 as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/hbase/meta/1588230740/rep_barrier/0627b1b20df14bae9a2900547c88dd87 2023-09-06T20:34:52,368 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/hbase/meta/1588230740/rep_barrier/0627b1b20df14bae9a2900547c88dd87, entries=12, sequenceid=183, filesize=6.5 K 2023-09-06T20:34:52,368 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/hbase/meta/1588230740/.tmp/table/a2aebfbff2e7471d8e7316265ed073f9 as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/hbase/meta/1588230740/table/a2aebfbff2e7471d8e7316265ed073f9 2023-09-06T20:34:52,373 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/hbase/meta/1588230740/table/a2aebfbff2e7471d8e7316265ed073f9, entries=32, sequenceid=183, filesize=7.3 K 2023-09-06T20:34:52,374 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~38.00 KB/38910, heapSize ~66.25 KB/67840, currentSize=0 B/0 for 1588230740 in 501ms, sequenceid=183, compaction requested=false 2023-09-06T20:34:52,383 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/hbase/meta/1588230740/recovered.edits/186.seqid, newMaxSeqId=186, maxSeqId=1 2023-09-06T20:34:52,384 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:34:52,384 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-09-06T20:34:52,384 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-09-06T20:34:52,384 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-09-06T20:34:52,385 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-09-06T20:34:52,474 DEBUG [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(1609): Waiting on 3a1cfaf5b9def8649351000383a2d6de, cf63a85c53279bc94811b50578f31281 2023-09-06T20:34:52,675 DEBUG [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(1609): Waiting on 3a1cfaf5b9def8649351000383a2d6de, cf63a85c53279bc94811b50578f31281 2023-09-06T20:34:52,745 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=717 B at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/4f4da85a052d4e598b288ef7a835be8d 2023-09-06T20:34:52,752 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/ddabe68ec3de4a738630c10caaf30f6e is 524, key is session:backup_1694032479233/session:context/1694032491797/Put/seqid=0 2023-09-06T20:34:52,756 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=775 B at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/ddabe68ec3de4a738630c10caaf30f6e 2023-09-06T20:34:52,761 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/meta/4f4da85a052d4e598b288ef7a835be8d as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/4f4da85a052d4e598b288ef7a835be8d 2023-09-06T20:34:52,765 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/meta/4f4da85a052d4e598b288ef7a835be8d, entries=6, sequenceid=148, filesize=5.8 K 2023-09-06T20:34:52,766 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/.tmp/session/ddabe68ec3de4a738630c10caaf30f6e as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/ddabe68ec3de4a738630c10caaf30f6e 2023-09-06T20:34:52,770 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/session/ddabe68ec3de4a738630c10caaf30f6e, entries=2, sequenceid=148, filesize=5.5 K 2023-09-06T20:34:52,771 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~1.46 KB/1492, heapSize ~2.58 KB/2640, currentSize=0 B/0 for cf63a85c53279bc94811b50578f31281 in 433ms, sequenceid=148, compaction requested=true 2023-09-06T20:34:52,790 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system/cf63a85c53279bc94811b50578f31281/recovered.edits/151.seqid, newMaxSeqId=151, maxSeqId=1 2023-09-06T20:34:52,790 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:34:52,791 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:34:52,791 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for cf63a85c53279bc94811b50578f31281: 2023-09-06T20:34:52,791 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1694032303949.cf63a85c53279bc94811b50578f31281. 2023-09-06T20:34:52,793 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 3a1cfaf5b9def8649351000383a2d6de, disabling compactions & flushes 2023-09-06T20:34:52,793 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1694032305079.3a1cfaf5b9def8649351000383a2d6de. 2023-09-06T20:34:52,793 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1694032305079.3a1cfaf5b9def8649351000383a2d6de. 2023-09-06T20:34:52,793 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1694032305079.3a1cfaf5b9def8649351000383a2d6de. after waiting 0 ms 2023-09-06T20:34:52,793 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1694032305079.3a1cfaf5b9def8649351000383a2d6de. 2023-09-06T20:34:52,793 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 3a1cfaf5b9def8649351000383a2d6de 2/2 column families, dataSize=5.66 KB heapSize=8.50 KB 2023-09-06T20:34:52,807 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system_bulk/3a1cfaf5b9def8649351000383a2d6de/.tmp/meta/3aa150e0d3c24355bc3286ebddc73d79 is 323, key is bulk:ns2:test-16940322964961:cd8331e491339ba2021603869fc044dc:5f020f30827940489104cc2b9277b6d8_SeqId_4_/meta:path/1694032323819/Put/seqid=0 2023-09-06T20:34:52,875 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(1579): Waiting on 1 regions to close 2023-09-06T20:34:52,875 DEBUG [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(1583): Online Regions={3a1cfaf5b9def8649351000383a2d6de=backup:system_bulk,,1694032305079.3a1cfaf5b9def8649351000383a2d6de.} 2023-09-06T20:34:52,875 DEBUG [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(1609): Waiting on 3a1cfaf5b9def8649351000383a2d6de 2023-09-06T20:34:53,075 DEBUG [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(1609): Waiting on 3a1cfaf5b9def8649351000383a2d6de 2023-09-06T20:34:53,211 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.66 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system_bulk/3a1cfaf5b9def8649351000383a2d6de/.tmp/meta/3aa150e0d3c24355bc3286ebddc73d79 2023-09-06T20:34:53,218 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system_bulk/3a1cfaf5b9def8649351000383a2d6de/.tmp/meta/3aa150e0d3c24355bc3286ebddc73d79 as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system_bulk/3a1cfaf5b9def8649351000383a2d6de/meta/3aa150e0d3c24355bc3286ebddc73d79 2023-09-06T20:34:53,226 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system_bulk/3a1cfaf5b9def8649351000383a2d6de/meta/3aa150e0d3c24355bc3286ebddc73d79, entries=16, sequenceid=12, filesize=8.1 K 2023-09-06T20:34:53,227 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~5.66 KB/5800, heapSize ~8.23 KB/8432, currentSize=0 B/0 for 3a1cfaf5b9def8649351000383a2d6de in 433ms, sequenceid=12, compaction requested=false 2023-09-06T20:34:53,232 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/data/backup/system_bulk/3a1cfaf5b9def8649351000383a2d6de/recovered.edits/15.seqid, newMaxSeqId=15, maxSeqId=1 2023-09-06T20:34:53,232 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:34:53,233 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1694032305079.3a1cfaf5b9def8649351000383a2d6de. 2023-09-06T20:34:53,233 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 3a1cfaf5b9def8649351000383a2d6de: 2023-09-06T20:34:53,233 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1694032305079.3a1cfaf5b9def8649351000383a2d6de. 2023-09-06T20:34:53,275 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,36961,1694032287907; all regions closed. 2023-09-06T20:34:53,288 DEBUG [RS:0;jenkins-hbase3:36961 {}] wal.AbstractFSWAL(1062): Moved 10 WAL file(s) to /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/oldWALs 2023-09-06T20:34:53,288 INFO [RS:0;jenkins-hbase3:36961 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C36961%2C1694032287907.meta:.meta(num 1694032481391) 2023-09-06T20:34:53,290 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/WALs/jenkins-hbase3.apache.org,36961,1694032287907/jenkins-hbase3.apache.org%2C36961%2C1694032287907.1694032481413 not finished, retry = 0 2023-09-06T20:34:53,397 DEBUG [RS:0;jenkins-hbase3:36961 {}] wal.AbstractFSWAL(1062): Moved 7 WAL file(s) to /user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/oldWALs 2023-09-06T20:34:53,397 INFO [RS:0;jenkins-hbase3:36961 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C36961%2C1694032287907:(num 1694032481413) 2023-09-06T20:34:53,397 DEBUG [RS:0;jenkins-hbase3:36961 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:53,397 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.LeaseManager(133): Closed leases 2023-09-06T20:34:53,397 INFO [RS:0;jenkins-hbase3:36961 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2023-09-06T20:34:53,397 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-09-06T20:34:53,399 INFO [RS:0;jenkins-hbase3:36961 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:36961 2023-09-06T20:34:53,408 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,36961,1694032287907] 2023-09-06T20:34:53,408 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,36961,1694032287907; numProcessing=1 2023-09-06T20:34:53,409 INFO [RegionServerTracker-0 {}] master.ServerManager(569): Cluster shutdown set; jenkins-hbase3.apache.org,36961,1694032287907 expired; onlineServers=0 2023-09-06T20:34:53,410 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,37419,1694032284546' ***** 2023-09-06T20:34:53,410 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2531): STOPPED: Cluster shutdown set; onlineServer=0 2023-09-06T20:34:53,410 DEBUG [M:0;jenkins-hbase3:37419 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33600fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-09-06T20:34:53,410 INFO [M:0;jenkins-hbase3:37419 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-09-06T20:34:53,455 INFO [M:0;jenkins-hbase3:37419 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@1f3fecf1{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-09-06T20:34:53,456 INFO [M:0;jenkins-hbase3:37419 {}] server.AbstractConnector(383): Stopped ServerConnector@6c0e7723{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-09-06T20:34:53,456 INFO [M:0;jenkins-hbase3:37419 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-09-06T20:34:53,456 INFO [M:0;jenkins-hbase3:37419 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@7c032c8b{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-09-06T20:34:53,457 INFO [M:0;jenkins-hbase3:37419 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@55008ff6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/hadoop.log.dir/,STOPPED} 2023-09-06T20:34:53,457 INFO [M:0;jenkins-hbase3:37419 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,37419,1694032284546 2023-09-06T20:34:53,457 INFO [M:0;jenkins-hbase3:37419 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,37419,1694032284546; all regions closed. 2023-09-06T20:34:53,457 DEBUG [M:0;jenkins-hbase3:37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:53,457 INFO [M:0;jenkins-hbase3:37419 {}] master.HMaster(1665): Stopping master jetty server 2023-09-06T20:34:53,458 INFO [M:0;jenkins-hbase3:37419 {}] server.AbstractConnector(383): Stopped ServerConnector@584f5459{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-09-06T20:34:53,458 DEBUG [M:0;jenkins-hbase3:37419 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-09-06T20:34:53,458 INFO [M:0;jenkins-hbase3:37419 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-09-06T20:34:53,458 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-09-06T20:34:53,459 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1694032289673 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1694032289673,5,FailOnTimeoutGroup] 2023-09-06T20:34:53,459 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1694032289673 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1694032289673,5,FailOnTimeoutGroup] 2023-09-06T20:34:53,459 DEBUG [M:0;jenkins-hbase3:37419 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:53,459 DEBUG [M:0;jenkins-hbase3:37419 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-09-06T20:34:53,459 INFO [M:0;jenkins-hbase3:37419 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-09-06T20:34:53,459 DEBUG [M:0;jenkins-hbase3:37419 {}] master.HMaster(1688): Stopping service threads 2023-09-06T20:34:53,459 INFO [M:0;jenkins-hbase3:37419 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-09-06T20:34:53,459 ERROR [M:0;jenkins-hbase3:37419 {}] procedure2.ProcedureExecutor(654): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-12,5,PEWorkerGroup] 2023-09-06T20:34:53,460 INFO [M:0;jenkins-hbase3:37419 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-09-06T20:34:53,460 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-09-06T20:34:53,460 WARN [M:0;jenkins-hbase3:37419 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-09-06T20:34:53,461 INFO [M:0;jenkins-hbase3:37419 {}] assignment.AssignmentManager(382): Stopping assignment manager 2023-09-06T20:34:53,461 INFO [M:0;jenkins-hbase3:37419 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-09-06T20:34:53,461 DEBUG [M:0;jenkins-hbase3:37419 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-09-06T20:34:53,472 INFO [M:0;jenkins-hbase3:37419 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:34:53,472 DEBUG [M:0;jenkins-hbase3:37419 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:34:53,472 DEBUG [M:0;jenkins-hbase3:37419 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-09-06T20:34:53,472 DEBUG [M:0;jenkins-hbase3:37419 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:34:53,472 INFO [M:0;jenkins-hbase3:37419 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=621.13 KB heapSize=755.41 KB 2023-09-06T20:34:53,507 DEBUG [M:0;jenkins-hbase3:37419 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d30f81489a064a02b27df0f7f2114055 is 88, key is hbase:meta,,1/info:sn/1694032290279/Put/seqid=0 2023-09-06T20:34:53,508 INFO [RS:0;jenkins-hbase3:36961 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,36961,1694032287907; zookeeper connection closed. 2023-09-06T20:34:53,508 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@628123a8 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@628123a8 2023-09-06T20:34:53,508 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-09-06T20:34:53,912 INFO [M:0;jenkins-hbase3:37419 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=1608 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d30f81489a064a02b27df0f7f2114055 2023-09-06T20:34:53,930 DEBUG [M:0;jenkins-hbase3:37419 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/17daa18a245e4697b18a835fccf6574d is 987, key is \x00\x00\x00\x00\x00\x00\x00\x1D/proc:d/1694032305872/Put/seqid=0 2023-09-06T20:34:54,334 INFO [M:0;jenkins-hbase3:37419 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=620.54 KB at sequenceid=1608 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/17daa18a245e4697b18a835fccf6574d 2023-09-06T20:34:54,340 INFO [M:0;jenkins-hbase3:37419 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 17daa18a245e4697b18a835fccf6574d 2023-09-06T20:34:54,352 DEBUG [M:0;jenkins-hbase3:37419 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8eb77027c88147ac91763f74ba92f501 is 82, key is jenkins-hbase3.apache.org,36961,1694032287907/rs:state/1694032289804/Put/seqid=0 2023-09-06T20:34:54,756 INFO [M:0;jenkins-hbase3:37419 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=1608 (bloomFilter=true), to=hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8eb77027c88147ac91763f74ba92f501 2023-09-06T20:34:54,760 DEBUG [M:0;jenkins-hbase3:37419 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d30f81489a064a02b27df0f7f2114055 as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d30f81489a064a02b27df0f7f2114055 2023-09-06T20:34:54,764 INFO [M:0;jenkins-hbase3:37419 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d30f81489a064a02b27df0f7f2114055, entries=8, sequenceid=1608, filesize=5.5 K 2023-09-06T20:34:54,765 DEBUG [M:0;jenkins-hbase3:37419 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/17daa18a245e4697b18a835fccf6574d as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/17daa18a245e4697b18a835fccf6574d 2023-09-06T20:34:54,769 INFO [M:0;jenkins-hbase3:37419 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 17daa18a245e4697b18a835fccf6574d 2023-09-06T20:34:54,769 INFO [M:0;jenkins-hbase3:37419 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/17daa18a245e4697b18a835fccf6574d, entries=173, sequenceid=1608, filesize=39.1 K 2023-09-06T20:34:54,770 DEBUG [M:0;jenkins-hbase3:37419 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8eb77027c88147ac91763f74ba92f501 as hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8eb77027c88147ac91763f74ba92f501 2023-09-06T20:34:54,775 INFO [M:0;jenkins-hbase3:37419 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39327/user/jenkins/test-data/1ede32cc-19d6-d2b6-506e-72cca011d3c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8eb77027c88147ac91763f74ba92f501, entries=1, sequenceid=1608, filesize=5.1 K 2023-09-06T20:34:54,776 INFO [M:0;jenkins-hbase3:37419 {}] regionserver.HRegion(3022): Finished flush of dataSize ~621.13 KB/636042, heapSize ~755.12 KB/773240, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1304ms, sequenceid=1608, compaction requested=false 2023-09-06T20:34:54,780 INFO [M:0;jenkins-hbase3:37419 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:34:54,780 DEBUG [M:0;jenkins-hbase3:37419 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-09-06T20:34:54,783 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-09-06T20:34:54,783 INFO [M:0;jenkins-hbase3:37419 {}] flush.MasterFlushTableProcedureManager(85): stop: server shutting down. 2023-09-06T20:34:54,783 INFO [M:0;jenkins-hbase3:37419 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-09-06T20:34:54,784 INFO [M:0;jenkins-hbase3:37419 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:37419 2023-09-06T20:34:54,887 INFO [M:0;jenkins-hbase3:37419 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,37419,1694032284546; zookeeper connection closed. 2023-09-06T20:34:54,896 WARN [Listener at jenkins-hbase3.apache.org/33095 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-09-06T20:34:54,900 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-09-06T20:34:55,008 WARN [BP-1247140837-172.31.12.81-1694032281151 heartbeating to localhost/127.0.0.1:39327 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-09-06T20:34:55,009 WARN [BP-1247140837-172.31.12.81-1694032281151 heartbeating to localhost/127.0.0.1:39327 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-1247140837-172.31.12.81-1694032281151 (Datanode Uuid 1d2bcda5-678a-442c-9e83-5e69f50490fe) service to localhost/127.0.0.1:39327 2023-09-06T20:34:55,011 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/cluster_26d71070-e599-b36c-2d51-b2ad8bb6051e/dfs/data/data1/current/BP-1247140837-172.31.12.81-1694032281151 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-09-06T20:34:55,011 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/6faa1fb1-2799-407b-37bd-694116e812a4/cluster_26d71070-e599-b36c-2d51-b2ad8bb6051e/dfs/data/data2/current/BP-1247140837-172.31.12.81-1694032281151 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-09-06T20:34:55,050 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-09-06T20:34:55,272 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-09-06T20:34:55,273 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-09-06T20:34:55,283 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-09-06T20:34:55,400 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-09-06T20:34:55,513 ERROR [Thread[Thread-275,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-09-06T20:34:55,513 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-09-06T20:34:55,620 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-09-06T20:34:55,628 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-09-06T20:34:55,629 ERROR [Thread[Thread-285,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-09-06T20:34:55,634 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-09-06T20:34:55,736 ERROR [Thread[Thread-250,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-09-06T20:34:55,737 INFO [Listener at jenkins-hbase3.apache.org/33095 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped