2023-08-10 06:33:29,491 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@13c9d689 2023-08-10 06:33:29,510 main DEBUG Took 0.015627 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-08-10 06:33:29,511 main DEBUG PluginManager 'Core' found 129 plugins 2023-08-10 06:33:29,511 main DEBUG PluginManager 'Level' found 0 plugins 2023-08-10 06:33:29,512 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-08-10 06:33:29,514 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:33:29,526 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-08-10 06:33:29,543 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:33:29,546 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:33:29,546 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:33:29,547 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:33:29,547 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:33:29,548 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:33:29,549 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:33:29,549 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:33:29,550 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:33:29,550 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:33:29,551 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:33:29,551 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:33:29,552 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:33:29,552 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:33:29,553 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:33:29,553 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:33:29,554 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:33:29,554 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:33:29,555 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:33:29,555 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:33:29,556 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:33:29,556 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:33:29,557 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:33:29,557 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:33:29,558 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:33:29,558 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-08-10 06:33:29,561 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:33:29,564 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-08-10 06:33:29,566 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-08-10 06:33:29,566 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-08-10 06:33:29,568 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-08-10 06:33:29,568 main DEBUG PluginManager 'Converter' found 47 plugins 2023-08-10 06:33:29,585 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-08-10 06:33:29,589 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-08-10 06:33:29,594 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-08-10 06:33:29,595 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-08-10 06:33:29,595 main DEBUG createAppenders(={Console}) 2023-08-10 06:33:29,596 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@13c9d689 initialized 2023-08-10 06:33:29,597 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@13c9d689 2023-08-10 06:33:29,597 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@13c9d689 OK. 2023-08-10 06:33:29,598 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-08-10 06:33:29,598 main DEBUG OutputStream closed 2023-08-10 06:33:29,599 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-08-10 06:33:29,599 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-08-10 06:33:29,599 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@15c43bd9 OK 2023-08-10 06:33:29,652 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-08-10 06:33:29,654 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-08-10 06:33:29,656 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-08-10 06:33:29,657 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-08-10 06:33:29,657 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-08-10 06:33:29,658 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-08-10 06:33:29,658 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-08-10 06:33:29,658 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-08-10 06:33:29,658 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-08-10 06:33:29,659 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-08-10 06:33:29,659 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-08-10 06:33:29,659 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-08-10 06:33:29,659 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-08-10 06:33:29,660 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-08-10 06:33:29,660 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-08-10 06:33:29,661 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-08-10 06:33:29,661 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-08-10 06:33:29,662 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-08-10 06:33:29,666 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-08-10 06:33:29,666 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@1b6e1eff) with optional ClassLoader: null 2023-08-10 06:33:29,666 main DEBUG Shutdown hook enabled. Registering a new one. 2023-08-10 06:33:29,667 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@1b6e1eff] started OK. 2023-08-10T06:33:29,697 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestBackupMultipleDeletes timeout: 13 mins 2023-08-10 06:33:29,701 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-08-10 06:33:29,701 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-08-10T06:33:30,050 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50 2023-08-10T06:33:30,055 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-08-10T06:33:30,058 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:33:30,059 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-08-10T06:33:30,062 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/cluster_727a6324-ee3a-516e-29f4-b0167fb1d918, deleteOnExit=true 2023-08-10T06:33:30,062 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-08-10T06:33:30,063 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/test.cache.data in system properties and HBase conf 2023-08-10T06:33:30,063 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/hadoop.tmp.dir in system properties and HBase conf 2023-08-10T06:33:30,064 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/hadoop.log.dir in system properties and HBase conf 2023-08-10T06:33:30,064 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/mapreduce.cluster.local.dir in system properties and HBase conf 2023-08-10T06:33:30,064 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-08-10T06:33:30,064 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-08-10T06:33:30,178 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-08-10T06:33:30,659 DEBUG [Time-limited test {}] fs.HFileSystem(308): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-08-10T06:33:30,667 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-08-10T06:33:30,667 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-08-10T06:33:30,668 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-08-10T06:33:30,668 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-08-10T06:33:30,669 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-08-10T06:33:30,669 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-08-10T06:33:30,670 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-08-10T06:33:30,670 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/dfs.journalnode.edits.dir in system properties and HBase conf 2023-08-10T06:33:30,671 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-08-10T06:33:30,671 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/nfs.dump.dir in system properties and HBase conf 2023-08-10T06:33:30,672 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/java.io.tmpdir in system properties and HBase conf 2023-08-10T06:33:30,672 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/dfs.journalnode.edits.dir in system properties and HBase conf 2023-08-10T06:33:30,673 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-08-10T06:33:30,673 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-08-10T06:33:31,269 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-08-10T06:33:31,274 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-08-10T06:33:31,609 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-08-10T06:33:31,760 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@71c172e4 via org.mortbay.log.Slf4jLog 2023-08-10T06:33:31,777 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-08-10T06:33:31,821 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-08-10T06:33:31,858 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/java.io.tmpdir/Jetty_localhost_35735_hdfs____.n17ktb/webapp 2023-08-10T06:33:32,025 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:35735 2023-08-10T06:33:32,052 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-08-10T06:33:32,053 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-08-10T06:33:32,695 WARN [Listener at localhost/41489 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-08-10T06:33:32,764 WARN [Listener at localhost/41489 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-08-10T06:33:32,784 WARN [Listener at localhost/41489 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-08-10T06:33:32,790 INFO [Listener at localhost/41489 {}] log.Slf4jLog(67): jetty-6.1.26 2023-08-10T06:33:32,797 INFO [Listener at localhost/41489 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/java.io.tmpdir/Jetty_localhost_40289_datanode____nh4c2m/webapp 2023-08-10T06:33:32,906 INFO [Listener at localhost/41489 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:40289 2023-08-10T06:33:33,181 WARN [Listener at localhost/46655 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-08-10T06:33:33,458 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-08-10T06:33:33,617 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x5120de7d83ef2a03: Processing first storage report for DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5 from datanode 5cdc99a8-415e-4832-89c3-7ec2d3ac0fbf 2023-08-10T06:33:33,619 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x5120de7d83ef2a03: from storage DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5 node DatanodeRegistration(127.0.0.1:38797, datanodeUuid=5cdc99a8-415e-4832-89c3-7ec2d3ac0fbf, infoPort=35075, infoSecurePort=0, ipcPort=46655, storageInfo=lv=-57;cid=testClusterID;nsid=713926900;c=1691649211357), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2023-08-10T06:33:33,619 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x5120de7d83ef2a03: Processing first storage report for DS-a61a93dd-727a-4e3e-b578-6b5592cb7de3 from datanode 5cdc99a8-415e-4832-89c3-7ec2d3ac0fbf 2023-08-10T06:33:33,619 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x5120de7d83ef2a03: from storage DS-a61a93dd-727a-4e3e-b578-6b5592cb7de3 node DatanodeRegistration(127.0.0.1:38797, datanodeUuid=5cdc99a8-415e-4832-89c3-7ec2d3ac0fbf, infoPort=35075, infoSecurePort=0, ipcPort=46655, storageInfo=lv=-57;cid=testClusterID;nsid=713926900;c=1691649211357), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-08-10T06:33:33,702 DEBUG [Listener at localhost/46655 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50 2023-08-10T06:33:33,813 INFO [Listener at localhost/46655 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/cluster_727a6324-ee3a-516e-29f4-b0167fb1d918/zookeeper_0, clientPort=59679, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/cluster_727a6324-ee3a-516e-29f4-b0167fb1d918/zookeeper_0/version-2, dataDirSize=424 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/cluster_727a6324-ee3a-516e-29f4-b0167fb1d918/zookeeper_0/version-2, dataLogSize=424 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, serverId=0 2023-08-10T06:33:33,831 INFO [Listener at localhost/46655 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=59679 2023-08-10T06:33:33,838 INFO [Listener at localhost/46655 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:33:33,839 INFO [Listener at localhost/46655 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:33:34,518 INFO [Listener at localhost/46655 {}] util.FSUtils(461): Created version file at hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f with version=8 2023-08-10T06:33:34,518 INFO [Listener at localhost/46655 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/hbase-staging 2023-08-10T06:33:34,536 DEBUG [Listener at localhost/46655 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-08-10T06:33:34,536 DEBUG [Listener at localhost/46655 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-08-10T06:33:34,536 DEBUG [Listener at localhost/46655 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-08-10T06:33:34,536 DEBUG [Listener at localhost/46655 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-08-10T06:33:34,755 DEBUG [Listener at localhost/46655 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-08-10T06:33:34,923 INFO [Listener at localhost/46655 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-08-10T06:33:35,600 INFO [Listener at localhost/46655 {}] client.ConnectionUtils(128): master/jenkins-hbase3:0 server-side Connection retries=45 2023-08-10T06:33:35,647 INFO [Listener at localhost/46655 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-08-10T06:33:35,648 INFO [Listener at localhost/46655 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-08-10T06:33:35,648 INFO [Listener at localhost/46655 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-08-10T06:33:35,648 INFO [Listener at localhost/46655 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-08-10T06:33:35,648 INFO [Listener at localhost/46655 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-08-10T06:33:35,801 INFO [Listener at localhost/46655 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-08-10T06:33:35,909 DEBUG [Listener at localhost/46655 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-08-10T06:33:35,915 INFO [Listener at localhost/46655 {}] ipc.NettyRpcServer(177): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-08-10T06:33:35,964 DEBUG [Listener at localhost/46655 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 27961 (auto-detected) 2023-08-10T06:33:35,966 DEBUG [Listener at localhost/46655 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:1c:ff:fe:e2:13:ea (auto-detected) 2023-08-10T06:33:36,004 INFO [Listener at localhost/46655 {}] ipc.NettyRpcServer(144): Bind to /172.31.12.81:37947 2023-08-10T06:33:36,019 INFO [Listener at localhost/46655 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:33:36,021 INFO [Listener at localhost/46655 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:33:36,041 INFO [Listener at localhost/46655 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:37947 connecting to ZooKeeper ensemble=127.0.0.1:59679 2023-08-10T06:33:36,088 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:379470x0, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-08-10T06:33:36,092 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:37947-0x10060efe5be0000 connected 2023-08-10T06:33:36,122 DEBUG [Listener at localhost/46655 {}] zookeeper.ZKUtil(113): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-08-10T06:33:36,125 DEBUG [Listener at localhost/46655 {}] zookeeper.ZKUtil(113): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-08-10T06:33:36,129 DEBUG [Listener at localhost/46655 {}] zookeeper.ZKUtil(113): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-08-10T06:33:36,144 DEBUG [Listener at localhost/46655 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37947 2023-08-10T06:33:36,145 DEBUG [Listener at localhost/46655 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37947 2023-08-10T06:33:36,146 DEBUG [Listener at localhost/46655 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37947 2023-08-10T06:33:36,147 DEBUG [Listener at localhost/46655 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37947 2023-08-10T06:33:36,150 DEBUG [Listener at localhost/46655 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37947 2023-08-10T06:33:36,205 INFO [Listener at localhost/46655 {}] log.Log(170): Logging initialized @7452ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-08-10T06:33:36,341 INFO [Listener at localhost/46655 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-08-10T06:33:36,342 INFO [Listener at localhost/46655 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-08-10T06:33:36,343 INFO [Listener at localhost/46655 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-08-10T06:33:36,345 INFO [Listener at localhost/46655 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-08-10T06:33:36,345 INFO [Listener at localhost/46655 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-08-10T06:33:36,345 INFO [Listener at localhost/46655 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-08-10T06:33:36,348 INFO [Listener at localhost/46655 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-08-10T06:33:36,500 INFO [Listener at localhost/46655 {}] http.HttpServer(1219): Jetty bound to port 41475 2023-08-10T06:33:36,503 INFO [Listener at localhost/46655 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-08-10T06:33:36,542 INFO [Listener at localhost/46655 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-08-10T06:33:36,545 INFO [Listener at localhost/46655 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@213b0a5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/hadoop.log.dir/,AVAILABLE} 2023-08-10T06:33:36,546 INFO [Listener at localhost/46655 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-08-10T06:33:36,546 INFO [Listener at localhost/46655 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@53671fc7{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-08-10T06:33:36,615 INFO [Listener at localhost/46655 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-08-10T06:33:36,628 INFO [Listener at localhost/46655 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-08-10T06:33:36,629 INFO [Listener at localhost/46655 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-08-10T06:33:36,631 INFO [Listener at localhost/46655 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2023-08-10T06:33:36,639 INFO [Listener at localhost/46655 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-08-10T06:33:37,820 INFO [Listener at localhost/46655 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@77ee37e5{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-08-10T06:33:37,832 INFO [Listener at localhost/46655 {}] server.AbstractConnector(333): Started ServerConnector@432d25ff{HTTP/1.1, (http/1.1)}{0.0.0.0:41475} 2023-08-10T06:33:37,832 INFO [Listener at localhost/46655 {}] server.Server(415): Started @9079ms 2023-08-10T06:33:37,836 INFO [Listener at localhost/46655 {}] master.HMaster(484): hbase.rootdir=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f, hbase.cluster.distributed=false 2023-08-10T06:33:37,900 INFO [Listener at localhost/46655 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase3:0 server-side Connection retries=45 2023-08-10T06:33:37,900 INFO [Listener at localhost/46655 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-08-10T06:33:37,900 INFO [Listener at localhost/46655 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-08-10T06:33:37,900 INFO [Listener at localhost/46655 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-08-10T06:33:37,900 INFO [Listener at localhost/46655 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-08-10T06:33:37,900 INFO [Listener at localhost/46655 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-08-10T06:33:37,909 INFO [Listener at localhost/46655 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-08-10T06:33:37,911 INFO [Listener at localhost/46655 {}] ipc.NettyRpcServer(177): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-08-10T06:33:37,913 INFO [Listener at localhost/46655 {}] ipc.NettyRpcServer(144): Bind to /172.31.12.81:44853 2023-08-10T06:33:37,916 INFO [Listener at localhost/46655 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-08-10T06:33:37,924 DEBUG [Listener at localhost/46655 {}] mob.MobFileCache(120): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-08-10T06:33:37,925 INFO [Listener at localhost/46655 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:33:37,929 INFO [Listener at localhost/46655 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:33:37,931 INFO [Listener at localhost/46655 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:44853 connecting to ZooKeeper ensemble=127.0.0.1:59679 2023-08-10T06:33:37,936 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:448530x0, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-08-10T06:33:37,937 DEBUG [Listener at localhost/46655 {}] zookeeper.ZKUtil(113): regionserver:448530x0, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-08-10T06:33:37,937 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:44853-0x10060efe5be0001 connected 2023-08-10T06:33:37,939 DEBUG [Listener at localhost/46655 {}] zookeeper.ZKUtil(113): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-08-10T06:33:37,940 DEBUG [Listener at localhost/46655 {}] zookeeper.ZKUtil(113): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-08-10T06:33:37,942 DEBUG [Listener at localhost/46655 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44853 2023-08-10T06:33:37,942 DEBUG [Listener at localhost/46655 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44853 2023-08-10T06:33:37,942 DEBUG [Listener at localhost/46655 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44853 2023-08-10T06:33:37,944 DEBUG [Listener at localhost/46655 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44853 2023-08-10T06:33:37,945 DEBUG [Listener at localhost/46655 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44853 2023-08-10T06:33:37,950 INFO [Listener at localhost/46655 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-08-10T06:33:37,950 INFO [Listener at localhost/46655 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-08-10T06:33:37,950 INFO [Listener at localhost/46655 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-08-10T06:33:37,951 INFO [Listener at localhost/46655 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-08-10T06:33:37,951 INFO [Listener at localhost/46655 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-08-10T06:33:37,951 INFO [Listener at localhost/46655 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-08-10T06:33:37,951 INFO [Listener at localhost/46655 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-08-10T06:33:37,953 INFO [Listener at localhost/46655 {}] http.HttpServer(1219): Jetty bound to port 45585 2023-08-10T06:33:37,954 INFO [Listener at localhost/46655 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-08-10T06:33:37,955 INFO [Listener at localhost/46655 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-08-10T06:33:37,956 INFO [Listener at localhost/46655 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@5f1c3a05{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/hadoop.log.dir/,AVAILABLE} 2023-08-10T06:33:37,956 INFO [Listener at localhost/46655 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-08-10T06:33:37,957 INFO [Listener at localhost/46655 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@5452a265{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-08-10T06:33:37,970 INFO [Listener at localhost/46655 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-08-10T06:33:37,971 INFO [Listener at localhost/46655 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-08-10T06:33:37,971 INFO [Listener at localhost/46655 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-08-10T06:33:37,972 INFO [Listener at localhost/46655 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-08-10T06:33:37,974 INFO [Listener at localhost/46655 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-08-10T06:33:37,979 INFO [Listener at localhost/46655 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@343d88aa{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-08-10T06:33:37,980 INFO [Listener at localhost/46655 {}] server.AbstractConnector(333): Started ServerConnector@276d566e{HTTP/1.1, (http/1.1)}{0.0.0.0:45585} 2023-08-10T06:33:37,980 INFO [Listener at localhost/46655 {}] server.Server(415): Started @9227ms 2023-08-10T06:33:37,986 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-08-10T06:33:37,993 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@696643d9{HTTP/1.1, (http/1.1)}{0.0.0.0:38443} 2023-08-10T06:33:37,994 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @9241ms 2023-08-10T06:33:37,994 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2387): Adding backup master ZNode /1/backup-masters/jenkins-hbase3.apache.org,37947,1691649214739 2023-08-10T06:33:38,010 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-08-10T06:33:38,010 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-08-10T06:33:38,012 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase3.apache.org,37947,1691649214739 2023-08-10T06:33:38,012 DEBUG [M:0;jenkins-hbase3:37947 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:37947 2023-08-10T06:33:38,032 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-08-10T06:33:38,032 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-08-10T06:33:38,032 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:33:38,033 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:33:38,034 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on existing znode=/1/master 2023-08-10T06:33:38,036 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on existing znode=/1/master 2023-08-10T06:33:38,036 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase3.apache.org,37947,1691649214739 from backup master directory 2023-08-10T06:33:38,040 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase3.apache.org,37947,1691649214739 2023-08-10T06:33:38,040 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-08-10T06:33:38,040 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-08-10T06:33:38,041 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-08-10T06:33:38,042 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,37947,1691649214739 2023-08-10T06:33:38,045 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-08-10T06:33:38,047 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-08-10T06:33:38,154 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(610): Created cluster ID file at hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/hbase.id with ID: e43503ac-931a-456f-b903-a9f24689b3e7 2023-08-10T06:33:38,248 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:33:38,270 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:33:38,270 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:33:38,347 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:33:38,349 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-08-10T06:33:38,373 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(270): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-08-10T06:33:38,373 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(276): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-08-10T06:33:38,374 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(285): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:283) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:310) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:950) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:573) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:570) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-08-10T06:33:38,387 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:950) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:573) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:570) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-08-10T06:33:38,389 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-08-10T06:33:38,474 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/MasterData/data/master/store 2023-08-10T06:33:38,504 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-08-10T06:33:38,505 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:33:38,506 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-08-10T06:33:38,506 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:33:38,506 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:33:38,506 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-08-10T06:33:38,506 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:33:38,507 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:33:38,507 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-08-10T06:33:38,509 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/MasterData/data/master/store/.initializing 2023-08-10T06:33:38,509 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/MasterData/WALs/jenkins-hbase3.apache.org,37947,1691649214739 2023-08-10T06:33:38,523 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-08-10T06:33:38,538 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C37947%2C1691649214739, suffix=, logDir=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/MasterData/WALs/jenkins-hbase3.apache.org,37947,1691649214739, archiveDir=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/MasterData/oldWALs, maxLogs=10 2023-08-10T06:33:38,568 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/MasterData/WALs/jenkins-hbase3.apache.org,37947,1691649214739/jenkins-hbase3.apache.org%2C37947%2C1691649214739.1691649218544, exclude list is [], retry=0 2023-08-10T06:33:38,594 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK] 2023-08-10T06:33:38,601 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:346) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:430) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:424) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:482) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:477) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:489) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-08-10T06:33:38,670 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/MasterData/WALs/jenkins-hbase3.apache.org,37947,1691649214739/jenkins-hbase3.apache.org%2C37947%2C1691649214739.1691649218544 2023-08-10T06:33:38,670 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK]] 2023-08-10T06:33:38,671 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:33:38,671 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:33:38,676 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:33:38,677 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:33:38,748 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:33:38,783 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-08-10T06:33:38,788 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:33:38,792 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:33:38,792 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:33:38,796 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-08-10T06:33:38,796 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:33:38,797 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:33:38,797 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:33:38,800 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-08-10T06:33:38,800 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:33:38,801 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:33:38,801 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:33:38,804 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-08-10T06:33:38,804 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:33:38,805 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:33:38,809 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:33:38,810 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:33:38,827 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-08-10T06:33:38,830 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:33:38,835 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:33:38,836 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10998083520, jitterRate=0.02427634596824646}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-08-10T06:33:38,844 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-08-10T06:33:38,845 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-08-10T06:33:38,881 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27510f56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:33:38,936 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(859): No meta location available on zookeeper, skip migrating... 2023-08-10T06:33:38,957 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-08-10T06:33:38,957 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(564): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-08-10T06:33:38,961 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-08-10T06:33:38,962 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(584): Recovered RegionProcedureStore lease in 1 msec 2023-08-10T06:33:38,968 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(598): Loaded RegionProcedureStore in 6 msec 2023-08-10T06:33:38,969 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-08-10T06:33:39,003 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-08-10T06:33:39,020 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-08-10T06:33:39,022 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-08-10T06:33:39,025 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(150): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-08-10T06:33:39,027 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-08-10T06:33:39,028 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-08-10T06:33:39,031 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-08-10T06:33:39,038 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-08-10T06:33:39,040 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-08-10T06:33:39,041 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-08-10T06:33:39,042 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-08-10T06:33:39,058 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-08-10T06:33:39,059 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-08-10T06:33:39,065 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-08-10T06:33:39,065 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-08-10T06:33:39,065 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:33:39,065 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:33:39,066 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(803): Active/primary master=jenkins-hbase3.apache.org,37947,1691649214739, sessionid=0x10060efe5be0000, setting cluster-up flag (Was=false) 2023-08-10T06:33:39,076 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-08-10T06:33:39,084 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:33:39,084 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:33:39,090 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-08-10T06:33:39,092 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,37947,1691649214739 2023-08-10T06:33:39,095 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:33:39,095 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:33:39,100 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-08-10T06:33:39,102 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,37947,1691649214739 2023-08-10T06:33:39,107 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:33:39,107 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:33:39,113 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-08-10T06:33:39,117 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,37947,1691649214739 2023-08-10T06:33:39,197 DEBUG [RS:0;jenkins-hbase3:44853 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:44853 2023-08-10T06:33:39,203 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(997): ClusterId : e43503ac-931a-456f-b903-a9f24689b3e7 2023-08-10T06:33:39,205 INFO [RS:0;jenkins-hbase3:44853 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-08-10T06:33:39,209 DEBUG [RS:0;jenkins-hbase3:44853 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-08-10T06:33:39,226 DEBUG [RS:0;jenkins-hbase3:44853 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-08-10T06:33:39,226 DEBUG [RS:0;jenkins-hbase3:44853 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-08-10T06:33:39,230 DEBUG [RS:0;jenkins-hbase3:44853 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-08-10T06:33:39,230 DEBUG [RS:0;jenkins-hbase3:44853 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-08-10T06:33:39,233 DEBUG [RS:0;jenkins-hbase3:44853 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-08-10T06:33:39,234 DEBUG [RS:0;jenkins-hbase3:44853 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ca79fc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:33:39,242 DEBUG [RS:0;jenkins-hbase3:44853 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d4b1666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-08-10T06:33:39,249 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-08-10T06:33:39,249 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-08-10T06:33:39,249 DEBUG [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-08-10T06:33:39,260 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(3023): reportForDuty to master=jenkins-hbase3.apache.org,37947,1691649214739 with isa=jenkins-hbase3.apache.org/172.31.12.81:44853, startcode=1691649217899 2023-08-10T06:33:39,287 DEBUG [RS:0;jenkins-hbase3:44853 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-08-10T06:33:39,305 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-08-10T06:33:39,318 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-08-10T06:33:39,325 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-08-10T06:33:39,336 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,37947,1691649214739 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-08-10T06:33:39,358 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-08-10T06:33:39,359 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-08-10T06:33:39,359 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-08-10T06:33:39,359 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-08-10T06:33:39,359 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-08-10T06:33:39,359 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:33:39,359 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-08-10T06:33:39,359 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:33:39,389 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:38605, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-08-10T06:33:39,400 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1691649249400 2023-08-10T06:33:39,404 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-08-10T06:33:39,407 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-08-10T06:33:39,408 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-08-10T06:33:39,409 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-08-10T06:33:39,414 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:33:39,416 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37947 {}] ipc.MetricsHBaseServer(144): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3180) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:588) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-08-10T06:33:39,415 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-08-10T06:33:39,423 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-08-10T06:33:39,423 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-08-10T06:33:39,425 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-08-10T06:33:39,425 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-08-10T06:33:39,426 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-08-10T06:33:39,426 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-08-10T06:33:39,432 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-08-10T06:33:39,435 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-08-10T06:33:39,435 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-08-10T06:33:39,454 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x53d0f342 to 127.0.0.1:59679 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:33:39,460 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-08-10T06:33:39,461 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f 2023-08-10T06:33:39,469 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64cce07e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:33:39,469 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-08-10T06:33:39,475 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-08-10T06:33:39,475 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-08-10T06:33:39,478 DEBUG [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(3042): Master is not running yet 2023-08-10T06:33:39,478 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1691649219478,5,FailOnTimeoutGroup] 2023-08-10T06:33:39,478 WARN [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 100 ms and then retrying. 2023-08-10T06:33:39,488 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1691649219478,5,FailOnTimeoutGroup] 2023-08-10T06:33:39,488 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-08-10T06:33:39,489 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1634): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-08-10T06:33:39,490 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-08-10T06:33:39,491 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-08-10T06:33:39,580 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(3023): reportForDuty to master=jenkins-hbase3.apache.org,37947,1691649214739 with isa=jenkins-hbase3.apache.org/172.31.12.81:44853, startcode=1691649217899 2023-08-10T06:33:39,585 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37947 {}] master.ServerManager(403): Registering regionserver=jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:33:39,597 DEBUG [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f 2023-08-10T06:33:39,597 DEBUG [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:41489 2023-08-10T06:33:39,597 DEBUG [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=41475 2023-08-10T06:33:39,603 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-08-10T06:33:39,603 DEBUG [RS:0;jenkins-hbase3:44853 {}] zookeeper.ZKUtil(111): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:33:39,604 WARN [RS:0;jenkins-hbase3:44853 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-08-10T06:33:39,604 INFO [RS:0;jenkins-hbase3:44853 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-08-10T06:33:39,604 DEBUG [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:33:39,608 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,44853,1691649217899] 2023-08-10T06:33:39,624 DEBUG [RS:0;jenkins-hbase3:44853 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-08-10T06:33:39,634 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-08-10T06:33:39,647 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-08-10T06:33:39,652 INFO [RS:0;jenkins-hbase3:44853 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-08-10T06:33:39,652 INFO [RS:0;jenkins-hbase3:44853 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-08-10T06:33:39,653 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-08-10T06:33:39,664 INFO [RS:0;jenkins-hbase3:44853 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-08-10T06:33:39,664 DEBUG [RS:0;jenkins-hbase3:44853 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:33:39,664 DEBUG [RS:0;jenkins-hbase3:44853 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:33:39,664 DEBUG [RS:0;jenkins-hbase3:44853 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:33:39,664 DEBUG [RS:0;jenkins-hbase3:44853 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:33:39,664 DEBUG [RS:0;jenkins-hbase3:44853 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:33:39,665 DEBUG [RS:0;jenkins-hbase3:44853 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-08-10T06:33:39,665 DEBUG [RS:0;jenkins-hbase3:44853 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:33:39,665 DEBUG [RS:0;jenkins-hbase3:44853 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:33:39,665 DEBUG [RS:0;jenkins-hbase3:44853 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:33:39,665 DEBUG [RS:0;jenkins-hbase3:44853 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:33:39,665 DEBUG [RS:0;jenkins-hbase3:44853 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:33:39,665 DEBUG [RS:0;jenkins-hbase3:44853 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-08-10T06:33:39,666 INFO [RS:0;jenkins-hbase3:44853 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-08-10T06:33:39,667 INFO [RS:0;jenkins-hbase3:44853 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-08-10T06:33:39,667 INFO [RS:0;jenkins-hbase3:44853 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-08-10T06:33:39,667 INFO [RS:0;jenkins-hbase3:44853 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-08-10T06:33:39,667 INFO [RS:0;jenkins-hbase3:44853 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,44853,1691649217899-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-08-10T06:33:39,685 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-08-10T06:33:39,689 INFO [RS:0;jenkins-hbase3:44853 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,44853,1691649217899-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-08-10T06:33:39,705 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,44853,1691649217899 started 2023-08-10T06:33:39,705 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,44853,1691649217899, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:44853, sessionid=0x10060efe5be0001 2023-08-10T06:33:39,705 DEBUG [RS:0;jenkins-hbase3:44853 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-08-10T06:33:39,705 DEBUG [RS:0;jenkins-hbase3:44853 {}] flush.RegionServerFlushTableProcedureManager(106): Start region server flush procedure manager jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:33:39,706 DEBUG [RS:0;jenkins-hbase3:44853 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,44853,1691649217899' 2023-08-10T06:33:39,706 DEBUG [RS:0;jenkins-hbase3:44853 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-08-10T06:33:39,706 DEBUG [RS:0;jenkins-hbase3:44853 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-08-10T06:33:39,707 DEBUG [RS:0;jenkins-hbase3:44853 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-08-10T06:33:39,707 DEBUG [RS:0;jenkins-hbase3:44853 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-08-10T06:33:39,707 DEBUG [RS:0;jenkins-hbase3:44853 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,44853,1691649217899' 2023-08-10T06:33:39,707 DEBUG [RS:0;jenkins-hbase3:44853 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:33:39,708 DEBUG [RS:0;jenkins-hbase3:44853 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:33:39,709 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-08-10T06:33:39,709 DEBUG [RS:0;jenkins-hbase3:44853 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-08-10T06:33:39,709 DEBUG [RS:0;jenkins-hbase3:44853 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-08-10T06:33:39,709 DEBUG [RS:0;jenkins-hbase3:44853 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:33:39,709 DEBUG [RS:0;jenkins-hbase3:44853 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,44853,1691649217899' 2023-08-10T06:33:39,709 DEBUG [RS:0;jenkins-hbase3:44853 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-08-10T06:33:39,710 DEBUG [RS:0;jenkins-hbase3:44853 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-08-10T06:33:39,710 DEBUG [RS:0;jenkins-hbase3:44853 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-08-10T06:33:39,711 INFO [RS:0;jenkins-hbase3:44853 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-08-10T06:33:39,711 INFO [RS:0;jenkins-hbase3:44853 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-08-10T06:33:39,821 INFO [RS:0;jenkins-hbase3:44853 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-08-10T06:33:39,824 INFO [RS:0;jenkins-hbase3:44853 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C44853%2C1691649217899, suffix=, logDir=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899, archiveDir=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs, maxLogs=32 2023-08-10T06:33:39,838 DEBUG [RS:0;jenkins-hbase3:44853 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649219826, exclude list is [], retry=0 2023-08-10T06:33:39,844 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK] 2023-08-10T06:33:39,849 INFO [RS:0;jenkins-hbase3:44853 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649219826 2023-08-10T06:33:39,849 DEBUG [RS:0;jenkins-hbase3:44853 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK]] 2023-08-10T06:33:39,903 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:33:39,906 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-08-10T06:33:39,908 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-08-10T06:33:39,908 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:33:39,909 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:33:39,909 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-08-10T06:33:39,911 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-08-10T06:33:39,911 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:33:39,912 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:33:39,912 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-08-10T06:33:39,914 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-08-10T06:33:39,915 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:33:39,915 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:33:39,917 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/hbase/meta/1588230740 2023-08-10T06:33:39,917 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/hbase/meta/1588230740 2023-08-10T06:33:39,920 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-08-10T06:33:39,921 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-08-10T06:33:39,925 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:33:39,925 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11386461440, jitterRate=0.060446858406066895}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-08-10T06:33:39,927 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-08-10T06:33:39,927 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-08-10T06:33:39,927 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-08-10T06:33:39,927 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-08-10T06:33:39,927 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-08-10T06:33:39,927 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-08-10T06:33:39,929 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-08-10T06:33:39,929 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-08-10T06:33:39,932 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-08-10T06:33:39,932 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-08-10T06:33:39,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-08-10T06:33:39,953 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-08-10T06:33:39,959 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-08-10T06:33:40,111 DEBUG [jenkins-hbase3:37947 {}] assignment.AssignmentManager(2303): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-08-10T06:33:40,120 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:33:40,125 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,44853,1691649217899, state=OPENING 2023-08-10T06:33:40,135 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-08-10T06:33:40,140 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:33:40,140 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:33:40,141 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-08-10T06:33:40,141 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-08-10T06:33:40,144 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,44853,1691649217899}] 2023-08-10T06:33:40,335 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(721): New admin connection to jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:33:40,338 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-08-10T06:33:40,341 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:58490, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-08-10T06:33:40,353 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-08-10T06:33:40,354 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-08-10T06:33:40,355 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-08-10T06:33:40,358 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta, suffix=.meta, logDir=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899, archiveDir=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs, maxLogs=32 2023-08-10T06:33:40,377 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649220360.meta, exclude list is [], retry=0 2023-08-10T06:33:40,382 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK] 2023-08-10T06:33:40,386 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649220360.meta 2023-08-10T06:33:40,386 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK]] 2023-08-10T06:33:40,386 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:33:40,392 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:33:40,393 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-08-10T06:33:40,409 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-08-10T06:33:40,409 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-08-10T06:33:40,415 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-08-10T06:33:40,415 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:33:40,415 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-08-10T06:33:40,416 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-08-10T06:33:40,418 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-08-10T06:33:40,420 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-08-10T06:33:40,420 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:33:40,421 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:33:40,421 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-08-10T06:33:40,422 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-08-10T06:33:40,422 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:33:40,423 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:33:40,423 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-08-10T06:33:40,425 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-08-10T06:33:40,425 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:33:40,426 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:33:40,427 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/hbase/meta/1588230740 2023-08-10T06:33:40,430 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/hbase/meta/1588230740 2023-08-10T06:33:40,433 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-08-10T06:33:40,436 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-08-10T06:33:40,437 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11829892000, jitterRate=0.10174454748630524}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-08-10T06:33:40,438 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-08-10T06:33:40,451 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2552): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1691649220327 2023-08-10T06:33:40,471 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2579): Finished post open deploy task for hbase:meta,,1.1588230740 2023-08-10T06:33:40,472 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-08-10T06:33:40,473 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:33:40,475 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,44853,1691649217899, state=OPEN 2023-08-10T06:33:40,477 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-08-10T06:33:40,477 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-08-10T06:33:40,478 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-08-10T06:33:40,478 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-08-10T06:33:40,482 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=3, resume processing ppid=2 2023-08-10T06:33:40,482 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,44853,1691649217899 in 333 msec 2023-08-10T06:33:40,492 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=2, resume processing ppid=1 2023-08-10T06:33:40,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 542 msec 2023-08-10T06:33:40,497 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.3040 sec 2023-08-10T06:33:40,498 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1062): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1691649220498, completionTime=-1 2023-08-10T06:33:40,498 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(830): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-08-10T06:33:40,498 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1629): Joining cluster... 2023-08-10T06:33:40,741 DEBUG [hconnection-0x7a45b7aa-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:33:40,746 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:58506, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:33:40,763 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1641): Number of RegionServers=1 2023-08-10T06:33:40,763 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1691649280763 2023-08-10T06:33:40,763 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1691649340763 2023-08-10T06:33:40,763 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1648): Joined the cluster in 264 msec 2023-08-10T06:33:40,789 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37947,1691649214739-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-08-10T06:33:40,790 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37947,1691649214739-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-08-10T06:33:40,790 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37947,1691649214739-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-08-10T06:33:40,792 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:37947, period=300000, unit=MILLISECONDS is enabled. 2023-08-10T06:33:40,792 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-08-10T06:33:40,798 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-08-10T06:33:40,812 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-08-10T06:33:40,814 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2367): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-08-10T06:33:40,825 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-08-10T06:33:40,829 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:33:40,832 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:33:40,833 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:33:40,855 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 2a873d0a0a9423038307a20a4f0e1113, NAME => 'hbase:namespace,,1691649220813.2a873d0a0a9423038307a20a4f0e1113.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f 2023-08-10T06:33:41,266 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1691649220813.2a873d0a0a9423038307a20a4f0e1113.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:33:41,267 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing 2a873d0a0a9423038307a20a4f0e1113, disabling compactions & flushes 2023-08-10T06:33:41,267 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1691649220813.2a873d0a0a9423038307a20a4f0e1113. 2023-08-10T06:33:41,267 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1691649220813.2a873d0a0a9423038307a20a4f0e1113. 2023-08-10T06:33:41,267 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1691649220813.2a873d0a0a9423038307a20a4f0e1113. after waiting 0 ms 2023-08-10T06:33:41,267 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1691649220813.2a873d0a0a9423038307a20a4f0e1113. 2023-08-10T06:33:41,267 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1691649220813.2a873d0a0a9423038307a20a4f0e1113. 2023-08-10T06:33:41,267 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for 2a873d0a0a9423038307a20a4f0e1113: 2023-08-10T06:33:41,270 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:33:41,284 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1691649220813.2a873d0a0a9423038307a20a4f0e1113.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1691649221272"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691649221272"}]},"ts":"1691649221272"} 2023-08-10T06:33:41,319 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:33:41,321 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:33:41,326 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649221321"}]},"ts":"1691649221321"} 2023-08-10T06:33:41,332 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-08-10T06:33:41,338 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=2a873d0a0a9423038307a20a4f0e1113, ASSIGN}] 2023-08-10T06:33:41,340 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=2a873d0a0a9423038307a20a4f0e1113, ASSIGN 2023-08-10T06:33:41,342 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=2a873d0a0a9423038307a20a4f0e1113, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44853,1691649217899; forceNewPlan=false, retain=false 2023-08-10T06:33:41,493 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=2a873d0a0a9423038307a20a4f0e1113, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:33:41,497 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 2a873d0a0a9423038307a20a4f0e1113, server=jenkins-hbase3.apache.org,44853,1691649217899}] 2023-08-10T06:33:41,657 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1691649220813.2a873d0a0a9423038307a20a4f0e1113. 2023-08-10T06:33:41,658 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => 2a873d0a0a9423038307a20a4f0e1113, NAME => 'hbase:namespace,,1691649220813.2a873d0a0a9423038307a20a4f0e1113.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:33:41,659 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:33:41,659 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 2a873d0a0a9423038307a20a4f0e1113 2023-08-10T06:33:41,659 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1691649220813.2a873d0a0a9423038307a20a4f0e1113.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:33:41,659 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for 2a873d0a0a9423038307a20a4f0e1113 2023-08-10T06:33:41,659 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for 2a873d0a0a9423038307a20a4f0e1113 2023-08-10T06:33:41,661 INFO [StoreOpener-2a873d0a0a9423038307a20a4f0e1113-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 2a873d0a0a9423038307a20a4f0e1113 2023-08-10T06:33:41,664 INFO [StoreOpener-2a873d0a0a9423038307a20a4f0e1113-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a873d0a0a9423038307a20a4f0e1113 columnFamilyName info 2023-08-10T06:33:41,664 DEBUG [StoreOpener-2a873d0a0a9423038307a20a4f0e1113-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:33:41,665 INFO [StoreOpener-2a873d0a0a9423038307a20a4f0e1113-1 {}] regionserver.HStore(324): Store=2a873d0a0a9423038307a20a4f0e1113/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:33:41,667 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/hbase/namespace/2a873d0a0a9423038307a20a4f0e1113 2023-08-10T06:33:41,667 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/hbase/namespace/2a873d0a0a9423038307a20a4f0e1113 2023-08-10T06:33:41,670 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for 2a873d0a0a9423038307a20a4f0e1113 2023-08-10T06:33:41,674 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/hbase/namespace/2a873d0a0a9423038307a20a4f0e1113/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:33:41,675 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened 2a873d0a0a9423038307a20a4f0e1113; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11616012320, jitterRate=0.08182545006275177}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:33:41,676 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for 2a873d0a0a9423038307a20a4f0e1113: 2023-08-10T06:33:41,678 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2552): Post open deploy tasks for hbase:namespace,,1691649220813.2a873d0a0a9423038307a20a4f0e1113., pid=6, masterSystemTime=1691649221650 2023-08-10T06:33:41,682 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2579): Finished post open deploy task for hbase:namespace,,1691649220813.2a873d0a0a9423038307a20a4f0e1113. 2023-08-10T06:33:41,682 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1691649220813.2a873d0a0a9423038307a20a4f0e1113. 2023-08-10T06:33:41,683 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=2a873d0a0a9423038307a20a4f0e1113, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:33:41,690 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=6, resume processing ppid=5 2023-08-10T06:33:41,690 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 2a873d0a0a9423038307a20a4f0e1113, server=jenkins-hbase3.apache.org,44853,1691649217899 in 190 msec 2023-08-10T06:33:41,693 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=5, resume processing ppid=4 2023-08-10T06:33:41,694 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=2a873d0a0a9423038307a20a4f0e1113, ASSIGN in 352 msec 2023-08-10T06:33:41,695 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:33:41,695 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649221695"}]},"ts":"1691649221695"} 2023-08-10T06:33:41,698 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-08-10T06:33:41,709 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:33:41,713 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 894 msec 2023-08-10T06:33:41,729 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-08-10T06:33:41,731 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-08-10T06:33:41,731 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:33:41,731 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:33:41,788 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-08-10T06:33:41,812 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:33:41,820 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 43 msec 2023-08-10T06:33:41,824 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-08-10T06:33:41,839 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:33:41,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 18 msec 2023-08-10T06:33:41,854 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-08-10T06:33:41,857 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-08-10T06:33:41,857 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1192): Master has completed initialization 3.815sec 2023-08-10T06:33:41,860 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-08-10T06:33:41,863 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-08-10T06:33:41,864 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-08-10T06:33:41,866 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-08-10T06:33:41,866 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-08-10T06:33:41,868 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37947,1691649214739-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-08-10T06:33:41,869 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37947,1691649214739-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-08-10T06:33:41,873 DEBUG [Listener at localhost/46655 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0fbc2dc4 to 127.0.0.1:59679 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:33:41,887 DEBUG [Listener at localhost/46655 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e6eaf7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:33:41,895 DEBUG [Listener at localhost/46655 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-08-10T06:33:41,895 DEBUG [Listener at localhost/46655 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-08-10T06:33:41,899 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1295): Balancer post startup initialization complete, took 0 seconds 2023-08-10T06:33:41,901 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-08-10T06:33:41,915 DEBUG [hconnection-0x77f2d805-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:33:41,933 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:58510, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:33:41,946 INFO [Listener at localhost/46655 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,37947,1691649214739 2023-08-10T06:33:41,946 INFO [Listener at localhost/46655 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-08-10T06:33:41,947 INFO [Listener at localhost/46655 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/test.cache.data in system properties and HBase conf 2023-08-10T06:33:41,947 INFO [Listener at localhost/46655 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/hadoop.tmp.dir in system properties and HBase conf 2023-08-10T06:33:41,947 INFO [Listener at localhost/46655 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/hadoop.log.dir in system properties and HBase conf 2023-08-10T06:33:41,947 INFO [Listener at localhost/46655 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/mapreduce.cluster.local.dir in system properties and HBase conf 2023-08-10T06:33:41,947 INFO [Listener at localhost/46655 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-08-10T06:33:41,947 INFO [Listener at localhost/46655 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-08-10T06:33:41,947 INFO [Listener at localhost/46655 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-08-10T06:33:41,948 INFO [Listener at localhost/46655 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-08-10T06:33:41,948 INFO [Listener at localhost/46655 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-08-10T06:33:41,948 INFO [Listener at localhost/46655 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-08-10T06:33:41,948 INFO [Listener at localhost/46655 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-08-10T06:33:41,948 INFO [Listener at localhost/46655 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-08-10T06:33:41,948 INFO [Listener at localhost/46655 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-08-10T06:33:41,948 INFO [Listener at localhost/46655 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/dfs.journalnode.edits.dir in system properties and HBase conf 2023-08-10T06:33:41,948 INFO [Listener at localhost/46655 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-08-10T06:33:41,949 INFO [Listener at localhost/46655 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/nfs.dump.dir in system properties and HBase conf 2023-08-10T06:33:41,949 INFO [Listener at localhost/46655 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/java.io.tmpdir in system properties and HBase conf 2023-08-10T06:33:41,949 INFO [Listener at localhost/46655 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/dfs.journalnode.edits.dir in system properties and HBase conf 2023-08-10T06:33:41,949 INFO [Listener at localhost/46655 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-08-10T06:33:41,949 INFO [Listener at localhost/46655 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-08-10T06:33:43,151 WARN [Thread-249 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-08-10T06:33:43,530 INFO [Thread-249 {}] log.Slf4jLog(67): jetty-6.1.26 2023-08-10T06:33:43,536 INFO [Thread-249 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_39243_jobhistory____.x17q3v/webapp 2023-08-10T06:33:43,710 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-08-10T06:33:43,710 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-08-10T06:33:43,710 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-08-10T06:33:43,714 INFO [Thread-249 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-08-10T06:33:43,804 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:33:43,932 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:33:44,262 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-08-10T06:33:44,313 INFO [Thread-249 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:39243 2023-08-10T06:33:44,647 WARN [Listener at jenkins-hbase3.apache.org/37619 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-08-10T06:33:44,752 WARN [Listener at jenkins-hbase3.apache.org/37619 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-08-10T06:33:44,752 INFO [Listener at jenkins-hbase3.apache.org/37619 {}] log.Slf4jLog(67): jetty-6.1.26 2023-08-10T06:33:44,763 INFO [Listener at jenkins-hbase3.apache.org/37619 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_41459_cluster____cun66g/webapp 2023-08-10T06:33:44,874 INFO [Listener at jenkins-hbase3.apache.org/37619 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-08-10T06:33:44,875 INFO [Listener at jenkins-hbase3.apache.org/37619 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-08-10T06:33:44,875 INFO [Listener at jenkins-hbase3.apache.org/37619 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-08-10T06:33:44,876 INFO [Listener at jenkins-hbase3.apache.org/37619 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-08-10T06:33:44,889 INFO [Listener at jenkins-hbase3.apache.org/37619 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:33:44,918 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-08-10T06:33:44,919 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-08-10T06:33:44,928 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-08-10T06:33:44,928 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-08-10T06:33:44,930 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:33:44,930 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-08-10T06:33:44,930 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-08-10T06:33:44,931 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-08-10T06:33:44,931 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-08-10T06:33:44,931 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-08-10T06:33:45,163 INFO [Listener at jenkins-hbase3.apache.org/37619 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:33:45,289 INFO [Listener at jenkins-hbase3.apache.org/37619 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:33:45,298 INFO [Listener at jenkins-hbase3.apache.org/37619 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:41459 2023-08-10T06:33:45,742 WARN [Listener at jenkins-hbase3.apache.org/38947 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-08-10T06:33:45,769 WARN [Listener at jenkins-hbase3.apache.org/38947 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-08-10T06:33:45,769 INFO [Listener at jenkins-hbase3.apache.org/38947 {}] log.Slf4jLog(67): jetty-6.1.26 2023-08-10T06:33:45,776 INFO [Listener at jenkins-hbase3.apache.org/38947 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_43469_node____howtz/webapp 2023-08-10T06:33:45,837 INFO [Listener at jenkins-hbase3.apache.org/38947 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-08-10T06:33:45,837 INFO [Listener at jenkins-hbase3.apache.org/38947 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-08-10T06:33:45,837 INFO [Listener at jenkins-hbase3.apache.org/38947 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-08-10T06:33:45,837 INFO [Listener at jenkins-hbase3.apache.org/38947 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-08-10T06:33:45,846 INFO [Listener at jenkins-hbase3.apache.org/38947 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:33:45,875 INFO [Listener at jenkins-hbase3.apache.org/38947 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:33:45,950 INFO [Listener at jenkins-hbase3.apache.org/38947 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:33:45,964 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:33:45,995 INFO [Listener at jenkins-hbase3.apache.org/38947 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:43469 2023-08-10T06:33:46,138 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-08-10T06:33:46,139 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-08-10T06:33:46,255 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-08-10T06:33:46,262 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-08-10T06:33:46,263 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] log.Slf4jLog(67): jetty-6.1.26 2023-08-10T06:33:46,273 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_43211_node____v6uiaa/webapp 2023-08-10T06:33:46,327 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-08-10T06:33:46,328 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-08-10T06:33:46,328 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-08-10T06:33:46,328 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-08-10T06:33:46,336 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:33:46,352 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:33:46,435 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:33:46,444 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:43211 2023-08-10T06:33:46,474 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-08-10T06:33:46,474 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:41489/backupUT 2023-08-10T06:33:46,487 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:33:46,491 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:33116, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:33:46,501 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns1'} 2023-08-10T06:33:46,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure2.ProcedureExecutor(1033): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-08-10T06:33:46,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=9 2023-08-10T06:33:46,528 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:33:46,534 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 31 msec 2023-08-10T06:33:46,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=9 2023-08-10T06:33:46,626 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns2'} 2023-08-10T06:33:46,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure2.ProcedureExecutor(1033): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-08-10T06:33:46,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=10 2023-08-10T06:33:46,639 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:33:46,647 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 18 msec 2023-08-10T06:33:46,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=10 2023-08-10T06:33:46,736 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns3'} 2023-08-10T06:33:46,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure2.ProcedureExecutor(1033): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-08-10T06:33:46,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=11 2023-08-10T06:33:46,750 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:33:46,755 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 16 msec 2023-08-10T06:33:46,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=11 2023-08-10T06:33:46,846 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns4'} 2023-08-10T06:33:46,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure2.ProcedureExecutor(1033): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-08-10T06:33:46,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=12 2023-08-10T06:33:46,859 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:33:46,866 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 18 msec 2023-08-10T06:33:46,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=12 2023-08-10T06:33:46,967 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'test-1691649226474', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:33:46,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure2.ProcedureExecutor(1033): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1691649226474 2023-08-10T06:33:46,972 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1691649226474 execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:33:46,973 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:33:46,974 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1691649226474 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:33:46,982 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "test-1691649226474" procId is: 13 2023-08-10T06:33:46,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-08-10T06:33:47,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-08-10T06:33:47,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-08-10T06:33:47,408 INFO [RegionOpenAndInit-test-1691649226474-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 3e5d3d5b3b7bc4e6a979a8585e22b81f, NAME => 'test-1691649226474,,1691649226962.3e5d3d5b3b7bc4e6a979a8585e22b81f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1691649226474', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f 2023-08-10T06:33:47,424 DEBUG [RegionOpenAndInit-test-1691649226474-pool-0 {}] regionserver.HRegion(888): Instantiated test-1691649226474,,1691649226962.3e5d3d5b3b7bc4e6a979a8585e22b81f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:33:47,424 DEBUG [RegionOpenAndInit-test-1691649226474-pool-0 {}] regionserver.HRegion(1668): Closing 3e5d3d5b3b7bc4e6a979a8585e22b81f, disabling compactions & flushes 2023-08-10T06:33:47,424 INFO [RegionOpenAndInit-test-1691649226474-pool-0 {}] regionserver.HRegion(1690): Closing region test-1691649226474,,1691649226962.3e5d3d5b3b7bc4e6a979a8585e22b81f. 2023-08-10T06:33:47,425 DEBUG [RegionOpenAndInit-test-1691649226474-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1691649226474,,1691649226962.3e5d3d5b3b7bc4e6a979a8585e22b81f. 2023-08-10T06:33:47,425 DEBUG [RegionOpenAndInit-test-1691649226474-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1691649226474,,1691649226962.3e5d3d5b3b7bc4e6a979a8585e22b81f. after waiting 0 ms 2023-08-10T06:33:47,425 DEBUG [RegionOpenAndInit-test-1691649226474-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1691649226474,,1691649226962.3e5d3d5b3b7bc4e6a979a8585e22b81f. 2023-08-10T06:33:47,425 INFO [RegionOpenAndInit-test-1691649226474-pool-0 {}] regionserver.HRegion(1909): Closed test-1691649226474,,1691649226962.3e5d3d5b3b7bc4e6a979a8585e22b81f. 2023-08-10T06:33:47,425 DEBUG [RegionOpenAndInit-test-1691649226474-pool-0 {}] regionserver.HRegion(1622): Region close journal for 3e5d3d5b3b7bc4e6a979a8585e22b81f: 2023-08-10T06:33:47,426 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1691649226474 execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:33:47,427 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1691649226474,,1691649226962.3e5d3d5b3b7bc4e6a979a8585e22b81f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1691649227426"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691649227426"}]},"ts":"1691649227426"} 2023-08-10T06:33:47,429 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:33:47,430 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1691649226474 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:33:47,430 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1691649226474","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649227430"}]},"ts":"1691649227430"} 2023-08-10T06:33:47,432 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1691649226474, state=ENABLING in hbase:meta 2023-08-10T06:33:47,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1691649226474, region=3e5d3d5b3b7bc4e6a979a8585e22b81f, ASSIGN}] 2023-08-10T06:33:47,438 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1691649226474, region=3e5d3d5b3b7bc4e6a979a8585e22b81f, ASSIGN 2023-08-10T06:33:47,439 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1691649226474, region=3e5d3d5b3b7bc4e6a979a8585e22b81f, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44853,1691649217899; forceNewPlan=false, retain=false 2023-08-10T06:33:47,591 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=3e5d3d5b3b7bc4e6a979a8585e22b81f, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:33:47,594 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure 3e5d3d5b3b7bc4e6a979a8585e22b81f, server=jenkins-hbase3.apache.org,44853,1691649217899}] 2023-08-10T06:33:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-08-10T06:33:47,756 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1691649226474,,1691649226962.3e5d3d5b3b7bc4e6a979a8585e22b81f. 2023-08-10T06:33:47,756 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => 3e5d3d5b3b7bc4e6a979a8585e22b81f, NAME => 'test-1691649226474,,1691649226962.3e5d3d5b3b7bc4e6a979a8585e22b81f.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:33:47,756 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:33:47,756 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1691649226474 3e5d3d5b3b7bc4e6a979a8585e22b81f 2023-08-10T06:33:47,757 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1691649226474,,1691649226962.3e5d3d5b3b7bc4e6a979a8585e22b81f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:33:47,757 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for 3e5d3d5b3b7bc4e6a979a8585e22b81f 2023-08-10T06:33:47,757 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for 3e5d3d5b3b7bc4e6a979a8585e22b81f 2023-08-10T06:33:47,760 INFO [StoreOpener-3e5d3d5b3b7bc4e6a979a8585e22b81f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 3e5d3d5b3b7bc4e6a979a8585e22b81f 2023-08-10T06:33:47,762 INFO [StoreOpener-3e5d3d5b3b7bc4e6a979a8585e22b81f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3e5d3d5b3b7bc4e6a979a8585e22b81f columnFamilyName f 2023-08-10T06:33:47,762 DEBUG [StoreOpener-3e5d3d5b3b7bc4e6a979a8585e22b81f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:33:47,763 INFO [StoreOpener-3e5d3d5b3b7bc4e6a979a8585e22b81f-1 {}] regionserver.HStore(324): Store=3e5d3d5b3b7bc4e6a979a8585e22b81f/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:33:47,765 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/default/test-1691649226474/3e5d3d5b3b7bc4e6a979a8585e22b81f 2023-08-10T06:33:47,766 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/default/test-1691649226474/3e5d3d5b3b7bc4e6a979a8585e22b81f 2023-08-10T06:33:47,770 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for 3e5d3d5b3b7bc4e6a979a8585e22b81f 2023-08-10T06:33:47,776 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/default/test-1691649226474/3e5d3d5b3b7bc4e6a979a8585e22b81f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:33:47,777 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened 3e5d3d5b3b7bc4e6a979a8585e22b81f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10696476480, jitterRate=-0.0038129985332489014}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:33:47,778 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for 3e5d3d5b3b7bc4e6a979a8585e22b81f: 2023-08-10T06:33:47,779 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2552): Post open deploy tasks for test-1691649226474,,1691649226962.3e5d3d5b3b7bc4e6a979a8585e22b81f., pid=15, masterSystemTime=1691649227747 2023-08-10T06:33:47,782 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2579): Finished post open deploy task for test-1691649226474,,1691649226962.3e5d3d5b3b7bc4e6a979a8585e22b81f. 2023-08-10T06:33:47,782 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1691649226474,,1691649226962.3e5d3d5b3b7bc4e6a979a8585e22b81f. 2023-08-10T06:33:47,783 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=3e5d3d5b3b7bc4e6a979a8585e22b81f, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:33:47,788 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=15, resume processing ppid=14 2023-08-10T06:33:47,788 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure 3e5d3d5b3b7bc4e6a979a8585e22b81f, server=jenkins-hbase3.apache.org,44853,1691649217899 in 191 msec 2023-08-10T06:33:47,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=14, resume processing ppid=13 2023-08-10T06:33:47,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1691649226474, region=3e5d3d5b3b7bc4e6a979a8585e22b81f, ASSIGN in 352 msec 2023-08-10T06:33:47,792 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1691649226474 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:33:47,793 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1691649226474","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649227792"}]},"ts":"1691649227792"} 2023-08-10T06:33:47,795 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1691649226474, state=ENABLED in hbase:meta 2023-08-10T06:33:47,798 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1691649226474 execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:33:47,800 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1691649226474 in 830 msec 2023-08-10T06:33:48,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-08-10T06:33:48,097 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: default:test-1691649226474, procId: 13 completed 2023-08-10T06:33:48,098 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4f75312a to 127.0.0.1:59679 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:33:48,103 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@223a0fcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:33:48,106 DEBUG [hconnection-0x4c960edd-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:33:48,108 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:58520, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:33:48,114 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44853 {}] regionserver.HRegion(8210): writing data to region test-1691649226474,,1691649226962.3e5d3d5b3b7bc4e6a979a8585e22b81f. with WAL disabled. Data may be lost in the event of a crash. 2023-08-10T06:33:48,222 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns2:test-16916492264741', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:33:48,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure2.ProcedureExecutor(1033): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-16916492264741 2023-08-10T06:33:48,226 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16916492264741 execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:33:48,226 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:33:48,226 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "test-16916492264741" procId is: 16 2023-08-10T06:33:48,227 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-16916492264741 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:33:48,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-08-10T06:33:48,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-08-10T06:33:48,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-08-10T06:33:48,644 INFO [RegionOpenAndInit-ns2:test-16916492264741-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 94f564ab80fb7e8519555fad2ae5463b, NAME => 'ns2:test-16916492264741,,1691649228221.94f564ab80fb7e8519555fad2ae5463b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16916492264741', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f 2023-08-10T06:33:48,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-08-10T06:33:49,058 DEBUG [RegionOpenAndInit-ns2:test-16916492264741-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16916492264741,,1691649228221.94f564ab80fb7e8519555fad2ae5463b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:33:49,058 DEBUG [RegionOpenAndInit-ns2:test-16916492264741-pool-0 {}] regionserver.HRegion(1668): Closing 94f564ab80fb7e8519555fad2ae5463b, disabling compactions & flushes 2023-08-10T06:33:49,058 INFO [RegionOpenAndInit-ns2:test-16916492264741-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16916492264741,,1691649228221.94f564ab80fb7e8519555fad2ae5463b. 2023-08-10T06:33:49,058 DEBUG [RegionOpenAndInit-ns2:test-16916492264741-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16916492264741,,1691649228221.94f564ab80fb7e8519555fad2ae5463b. 2023-08-10T06:33:49,058 DEBUG [RegionOpenAndInit-ns2:test-16916492264741-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16916492264741,,1691649228221.94f564ab80fb7e8519555fad2ae5463b. after waiting 0 ms 2023-08-10T06:33:49,058 DEBUG [RegionOpenAndInit-ns2:test-16916492264741-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16916492264741,,1691649228221.94f564ab80fb7e8519555fad2ae5463b. 2023-08-10T06:33:49,058 INFO [RegionOpenAndInit-ns2:test-16916492264741-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16916492264741,,1691649228221.94f564ab80fb7e8519555fad2ae5463b. 2023-08-10T06:33:49,058 DEBUG [RegionOpenAndInit-ns2:test-16916492264741-pool-0 {}] regionserver.HRegion(1622): Region close journal for 94f564ab80fb7e8519555fad2ae5463b: 2023-08-10T06:33:49,060 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-16916492264741 execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:33:49,060 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16916492264741,,1691649228221.94f564ab80fb7e8519555fad2ae5463b.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1691649229060"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691649229060"}]},"ts":"1691649229060"} 2023-08-10T06:33:49,063 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:33:49,064 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-16916492264741 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:33:49,064 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16916492264741","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649229064"}]},"ts":"1691649229064"} 2023-08-10T06:33:49,066 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16916492264741, state=ENABLING in hbase:meta 2023-08-10T06:33:49,071 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16916492264741, region=94f564ab80fb7e8519555fad2ae5463b, ASSIGN}] 2023-08-10T06:33:49,073 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16916492264741, region=94f564ab80fb7e8519555fad2ae5463b, ASSIGN 2023-08-10T06:33:49,074 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16916492264741, region=94f564ab80fb7e8519555fad2ae5463b, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44853,1691649217899; forceNewPlan=false, retain=false 2023-08-10T06:33:49,226 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=94f564ab80fb7e8519555fad2ae5463b, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:33:49,229 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure 94f564ab80fb7e8519555fad2ae5463b, server=jenkins-hbase3.apache.org,44853,1691649217899}] 2023-08-10T06:33:49,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-08-10T06:33:49,386 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-16916492264741,,1691649228221.94f564ab80fb7e8519555fad2ae5463b. 2023-08-10T06:33:49,386 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => 94f564ab80fb7e8519555fad2ae5463b, NAME => 'ns2:test-16916492264741,,1691649228221.94f564ab80fb7e8519555fad2ae5463b.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:33:49,387 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:33:49,387 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16916492264741 94f564ab80fb7e8519555fad2ae5463b 2023-08-10T06:33:49,387 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-16916492264741,,1691649228221.94f564ab80fb7e8519555fad2ae5463b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:33:49,387 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for 94f564ab80fb7e8519555fad2ae5463b 2023-08-10T06:33:49,387 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for 94f564ab80fb7e8519555fad2ae5463b 2023-08-10T06:33:49,389 INFO [StoreOpener-94f564ab80fb7e8519555fad2ae5463b-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 94f564ab80fb7e8519555fad2ae5463b 2023-08-10T06:33:49,391 INFO [StoreOpener-94f564ab80fb7e8519555fad2ae5463b-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94f564ab80fb7e8519555fad2ae5463b columnFamilyName f 2023-08-10T06:33:49,391 DEBUG [StoreOpener-94f564ab80fb7e8519555fad2ae5463b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:33:49,391 INFO [StoreOpener-94f564ab80fb7e8519555fad2ae5463b-1 {}] regionserver.HStore(324): Store=94f564ab80fb7e8519555fad2ae5463b/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:33:49,392 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/ns2/test-16916492264741/94f564ab80fb7e8519555fad2ae5463b 2023-08-10T06:33:49,393 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/ns2/test-16916492264741/94f564ab80fb7e8519555fad2ae5463b 2023-08-10T06:33:49,396 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for 94f564ab80fb7e8519555fad2ae5463b 2023-08-10T06:33:49,405 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/ns2/test-16916492264741/94f564ab80fb7e8519555fad2ae5463b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:33:49,406 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened 94f564ab80fb7e8519555fad2ae5463b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10689288320, jitterRate=-0.004482448101043701}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:33:49,407 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for 94f564ab80fb7e8519555fad2ae5463b: 2023-08-10T06:33:49,408 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2552): Post open deploy tasks for ns2:test-16916492264741,,1691649228221.94f564ab80fb7e8519555fad2ae5463b., pid=18, masterSystemTime=1691649229381 2023-08-10T06:33:49,411 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2579): Finished post open deploy task for ns2:test-16916492264741,,1691649228221.94f564ab80fb7e8519555fad2ae5463b. 2023-08-10T06:33:49,411 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-16916492264741,,1691649228221.94f564ab80fb7e8519555fad2ae5463b. 2023-08-10T06:33:49,411 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=94f564ab80fb7e8519555fad2ae5463b, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:33:49,417 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=18, resume processing ppid=17 2023-08-10T06:33:49,417 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure 94f564ab80fb7e8519555fad2ae5463b, server=jenkins-hbase3.apache.org,44853,1691649217899 in 185 msec 2023-08-10T06:33:49,419 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=17, resume processing ppid=16 2023-08-10T06:33:49,419 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16916492264741, region=94f564ab80fb7e8519555fad2ae5463b, ASSIGN in 346 msec 2023-08-10T06:33:49,420 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-16916492264741 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:33:49,421 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16916492264741","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649229421"}]},"ts":"1691649229421"} 2023-08-10T06:33:49,423 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16916492264741, state=ENABLED in hbase:meta 2023-08-10T06:33:49,426 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16916492264741 execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:33:49,428 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-16916492264741 in 1.2040 sec 2023-08-10T06:33:50,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-08-10T06:33:50,339 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns2:test-16916492264741, procId: 16 completed 2023-08-10T06:33:50,343 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44853 {}] regionserver.HRegion(8210): writing data to region ns2:test-16916492264741,,1691649228221.94f564ab80fb7e8519555fad2ae5463b. with WAL disabled. Data may be lost in the event of a crash. 2023-08-10T06:33:50,429 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns3:test-16916492264742', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:33:50,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure2.ProcedureExecutor(1033): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-16916492264742 2023-08-10T06:33:50,433 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16916492264742 execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:33:50,433 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "test-16916492264742" procId is: 19 2023-08-10T06:33:50,433 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:33:50,434 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-16916492264742 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:33:50,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-08-10T06:33:50,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-08-10T06:33:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-08-10T06:33:50,851 INFO [RegionOpenAndInit-ns3:test-16916492264742-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => a9e9ef626a68c685245826f3ca7f958f, NAME => 'ns3:test-16916492264742,,1691649230428.a9e9ef626a68c685245826f3ca7f958f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16916492264742', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f 2023-08-10T06:33:51,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-08-10T06:33:51,268 DEBUG [RegionOpenAndInit-ns3:test-16916492264742-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16916492264742,,1691649230428.a9e9ef626a68c685245826f3ca7f958f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:33:51,268 DEBUG [RegionOpenAndInit-ns3:test-16916492264742-pool-0 {}] regionserver.HRegion(1668): Closing a9e9ef626a68c685245826f3ca7f958f, disabling compactions & flushes 2023-08-10T06:33:51,268 INFO [RegionOpenAndInit-ns3:test-16916492264742-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16916492264742,,1691649230428.a9e9ef626a68c685245826f3ca7f958f. 2023-08-10T06:33:51,268 DEBUG [RegionOpenAndInit-ns3:test-16916492264742-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16916492264742,,1691649230428.a9e9ef626a68c685245826f3ca7f958f. 2023-08-10T06:33:51,268 DEBUG [RegionOpenAndInit-ns3:test-16916492264742-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16916492264742,,1691649230428.a9e9ef626a68c685245826f3ca7f958f. after waiting 0 ms 2023-08-10T06:33:51,268 DEBUG [RegionOpenAndInit-ns3:test-16916492264742-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16916492264742,,1691649230428.a9e9ef626a68c685245826f3ca7f958f. 2023-08-10T06:33:51,268 INFO [RegionOpenAndInit-ns3:test-16916492264742-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16916492264742,,1691649230428.a9e9ef626a68c685245826f3ca7f958f. 2023-08-10T06:33:51,268 DEBUG [RegionOpenAndInit-ns3:test-16916492264742-pool-0 {}] regionserver.HRegion(1622): Region close journal for a9e9ef626a68c685245826f3ca7f958f: 2023-08-10T06:33:51,270 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-16916492264742 execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:33:51,270 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16916492264742,,1691649230428.a9e9ef626a68c685245826f3ca7f958f.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1691649231270"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691649231270"}]},"ts":"1691649231270"} 2023-08-10T06:33:51,273 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:33:51,274 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-16916492264742 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:33:51,275 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16916492264742","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649231275"}]},"ts":"1691649231275"} 2023-08-10T06:33:51,277 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16916492264742, state=ENABLING in hbase:meta 2023-08-10T06:33:51,288 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16916492264742, region=a9e9ef626a68c685245826f3ca7f958f, ASSIGN}] 2023-08-10T06:33:51,292 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16916492264742, region=a9e9ef626a68c685245826f3ca7f958f, ASSIGN 2023-08-10T06:33:51,293 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16916492264742, region=a9e9ef626a68c685245826f3ca7f958f, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44853,1691649217899; forceNewPlan=false, retain=false 2023-08-10T06:33:51,444 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=a9e9ef626a68c685245826f3ca7f958f, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:33:51,447 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure a9e9ef626a68c685245826f3ca7f958f, server=jenkins-hbase3.apache.org,44853,1691649217899}] 2023-08-10T06:33:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-08-10T06:33:51,604 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-16916492264742,,1691649230428.a9e9ef626a68c685245826f3ca7f958f. 2023-08-10T06:33:51,605 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => a9e9ef626a68c685245826f3ca7f958f, NAME => 'ns3:test-16916492264742,,1691649230428.a9e9ef626a68c685245826f3ca7f958f.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:33:51,605 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:33:51,605 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16916492264742 a9e9ef626a68c685245826f3ca7f958f 2023-08-10T06:33:51,605 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-16916492264742,,1691649230428.a9e9ef626a68c685245826f3ca7f958f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:33:51,605 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for a9e9ef626a68c685245826f3ca7f958f 2023-08-10T06:33:51,605 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for a9e9ef626a68c685245826f3ca7f958f 2023-08-10T06:33:51,607 INFO [StoreOpener-a9e9ef626a68c685245826f3ca7f958f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region a9e9ef626a68c685245826f3ca7f958f 2023-08-10T06:33:51,609 INFO [StoreOpener-a9e9ef626a68c685245826f3ca7f958f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9e9ef626a68c685245826f3ca7f958f columnFamilyName f 2023-08-10T06:33:51,609 DEBUG [StoreOpener-a9e9ef626a68c685245826f3ca7f958f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:33:51,610 INFO [StoreOpener-a9e9ef626a68c685245826f3ca7f958f-1 {}] regionserver.HStore(324): Store=a9e9ef626a68c685245826f3ca7f958f/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:33:51,611 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/ns3/test-16916492264742/a9e9ef626a68c685245826f3ca7f958f 2023-08-10T06:33:51,611 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/ns3/test-16916492264742/a9e9ef626a68c685245826f3ca7f958f 2023-08-10T06:33:51,614 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for a9e9ef626a68c685245826f3ca7f958f 2023-08-10T06:33:51,617 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/ns3/test-16916492264742/a9e9ef626a68c685245826f3ca7f958f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:33:51,618 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened a9e9ef626a68c685245826f3ca7f958f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11803889760, jitterRate=0.09932290017604828}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:33:51,619 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for a9e9ef626a68c685245826f3ca7f958f: 2023-08-10T06:33:51,620 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2552): Post open deploy tasks for ns3:test-16916492264742,,1691649230428.a9e9ef626a68c685245826f3ca7f958f., pid=21, masterSystemTime=1691649231600 2023-08-10T06:33:51,622 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2579): Finished post open deploy task for ns3:test-16916492264742,,1691649230428.a9e9ef626a68c685245826f3ca7f958f. 2023-08-10T06:33:51,622 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-16916492264742,,1691649230428.a9e9ef626a68c685245826f3ca7f958f. 2023-08-10T06:33:51,623 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=a9e9ef626a68c685245826f3ca7f958f, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:33:51,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=21, resume processing ppid=20 2023-08-10T06:33:51,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure a9e9ef626a68c685245826f3ca7f958f, server=jenkins-hbase3.apache.org,44853,1691649217899 in 178 msec 2023-08-10T06:33:51,630 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=20, resume processing ppid=19 2023-08-10T06:33:51,630 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16916492264742, region=a9e9ef626a68c685245826f3ca7f958f, ASSIGN in 340 msec 2023-08-10T06:33:51,631 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-16916492264742 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:33:51,631 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16916492264742","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649231631"}]},"ts":"1691649231631"} 2023-08-10T06:33:51,633 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16916492264742, state=ENABLED in hbase:meta 2023-08-10T06:33:51,635 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16916492264742 execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:33:51,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-16916492264742 in 1.2060 sec 2023-08-10T06:33:52,141 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-16916492264741' 2023-08-10T06:33:52,142 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16916492264742' 2023-08-10T06:33:52,142 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1691649226474' 2023-08-10T06:33:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-08-10T06:33:52,544 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns3:test-16916492264742, procId: 19 completed 2023-08-10T06:33:52,544 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-16916492264742 get assigned. Timeout = 60000ms 2023-08-10T06:33:52,546 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-08-10T06:33:52,550 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-16916492264742 assigned to meta. Checking AM states. 2023-08-10T06:33:52,551 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-08-10T06:33:52,551 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-16916492264742 assigned. 2023-08-10T06:33:52,553 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns4:test-16916492264743', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:33:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure2.ProcedureExecutor(1033): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-16916492264743 2023-08-10T06:33:52,556 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16916492264743 execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:33:52,557 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:33:52,557 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns4" qualifier: "test-16916492264743" procId is: 22 2023-08-10T06:33:52,558 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-16916492264743 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:33:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-08-10T06:33:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-08-10T06:33:52,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-08-10T06:33:52,972 INFO [RegionOpenAndInit-ns4:test-16916492264743-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 930ae4244d4cd65da422c67705de93c0, NAME => 'ns4:test-16916492264743,,1691649232552.930ae4244d4cd65da422c67705de93c0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-16916492264743', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f 2023-08-10T06:33:53,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-08-10T06:33:53,263 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:33:53,386 DEBUG [RegionOpenAndInit-ns4:test-16916492264743-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-16916492264743,,1691649232552.930ae4244d4cd65da422c67705de93c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:33:53,386 DEBUG [RegionOpenAndInit-ns4:test-16916492264743-pool-0 {}] regionserver.HRegion(1668): Closing 930ae4244d4cd65da422c67705de93c0, disabling compactions & flushes 2023-08-10T06:33:53,386 INFO [RegionOpenAndInit-ns4:test-16916492264743-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-16916492264743,,1691649232552.930ae4244d4cd65da422c67705de93c0. 2023-08-10T06:33:53,386 DEBUG [RegionOpenAndInit-ns4:test-16916492264743-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16916492264743,,1691649232552.930ae4244d4cd65da422c67705de93c0. 2023-08-10T06:33:53,386 DEBUG [RegionOpenAndInit-ns4:test-16916492264743-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16916492264743,,1691649232552.930ae4244d4cd65da422c67705de93c0. after waiting 0 ms 2023-08-10T06:33:53,386 DEBUG [RegionOpenAndInit-ns4:test-16916492264743-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16916492264743,,1691649232552.930ae4244d4cd65da422c67705de93c0. 2023-08-10T06:33:53,386 INFO [RegionOpenAndInit-ns4:test-16916492264743-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-16916492264743,,1691649232552.930ae4244d4cd65da422c67705de93c0. 2023-08-10T06:33:53,386 DEBUG [RegionOpenAndInit-ns4:test-16916492264743-pool-0 {}] regionserver.HRegion(1622): Region close journal for 930ae4244d4cd65da422c67705de93c0: 2023-08-10T06:33:53,388 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-16916492264743 execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:33:53,388 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-16916492264743,,1691649232552.930ae4244d4cd65da422c67705de93c0.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1691649233388"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691649233388"}]},"ts":"1691649233388"} 2023-08-10T06:33:53,391 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:33:53,392 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-16916492264743 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:33:53,393 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16916492264743","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649233392"}]},"ts":"1691649233392"} 2023-08-10T06:33:53,395 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16916492264743, state=ENABLING in hbase:meta 2023-08-10T06:33:53,400 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16916492264743, region=930ae4244d4cd65da422c67705de93c0, ASSIGN}] 2023-08-10T06:33:53,404 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16916492264743, region=930ae4244d4cd65da422c67705de93c0, ASSIGN 2023-08-10T06:33:53,406 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-16916492264743, region=930ae4244d4cd65da422c67705de93c0, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44853,1691649217899; forceNewPlan=false, retain=false 2023-08-10T06:33:53,557 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=930ae4244d4cd65da422c67705de93c0, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:33:53,560 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure 930ae4244d4cd65da422c67705de93c0, server=jenkins-hbase3.apache.org,44853,1691649217899}] 2023-08-10T06:33:53,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-08-10T06:33:53,717 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-16916492264743,,1691649232552.930ae4244d4cd65da422c67705de93c0. 2023-08-10T06:33:53,717 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => 930ae4244d4cd65da422c67705de93c0, NAME => 'ns4:test-16916492264743,,1691649232552.930ae4244d4cd65da422c67705de93c0.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:33:53,718 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:33:53,718 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16916492264743 930ae4244d4cd65da422c67705de93c0 2023-08-10T06:33:53,718 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-16916492264743,,1691649232552.930ae4244d4cd65da422c67705de93c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:33:53,718 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for 930ae4244d4cd65da422c67705de93c0 2023-08-10T06:33:53,718 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for 930ae4244d4cd65da422c67705de93c0 2023-08-10T06:33:53,720 INFO [StoreOpener-930ae4244d4cd65da422c67705de93c0-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 930ae4244d4cd65da422c67705de93c0 2023-08-10T06:33:53,721 INFO [StoreOpener-930ae4244d4cd65da422c67705de93c0-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 930ae4244d4cd65da422c67705de93c0 columnFamilyName f 2023-08-10T06:33:53,722 DEBUG [StoreOpener-930ae4244d4cd65da422c67705de93c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:33:53,722 INFO [StoreOpener-930ae4244d4cd65da422c67705de93c0-1 {}] regionserver.HStore(324): Store=930ae4244d4cd65da422c67705de93c0/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:33:53,723 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/ns4/test-16916492264743/930ae4244d4cd65da422c67705de93c0 2023-08-10T06:33:53,724 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/ns4/test-16916492264743/930ae4244d4cd65da422c67705de93c0 2023-08-10T06:33:53,727 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for 930ae4244d4cd65da422c67705de93c0 2023-08-10T06:33:53,730 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/ns4/test-16916492264743/930ae4244d4cd65da422c67705de93c0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:33:53,731 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened 930ae4244d4cd65da422c67705de93c0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11181633120, jitterRate=0.041370734572410583}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:33:53,731 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for 930ae4244d4cd65da422c67705de93c0: 2023-08-10T06:33:53,732 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2552): Post open deploy tasks for ns4:test-16916492264743,,1691649232552.930ae4244d4cd65da422c67705de93c0., pid=24, masterSystemTime=1691649233712 2023-08-10T06:33:53,735 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2579): Finished post open deploy task for ns4:test-16916492264743,,1691649232552.930ae4244d4cd65da422c67705de93c0. 2023-08-10T06:33:53,735 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-16916492264743,,1691649232552.930ae4244d4cd65da422c67705de93c0. 2023-08-10T06:33:53,736 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=930ae4244d4cd65da422c67705de93c0, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:33:53,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=24, resume processing ppid=23 2023-08-10T06:33:53,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure 930ae4244d4cd65da422c67705de93c0, server=jenkins-hbase3.apache.org,44853,1691649217899 in 178 msec 2023-08-10T06:33:53,743 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=23, resume processing ppid=22 2023-08-10T06:33:53,743 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-16916492264743, region=930ae4244d4cd65da422c67705de93c0, ASSIGN in 340 msec 2023-08-10T06:33:53,744 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-16916492264743 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:33:53,744 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16916492264743","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649233744"}]},"ts":"1691649233744"} 2023-08-10T06:33:53,745 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16916492264743, state=ENABLED in hbase:meta 2023-08-10T06:33:53,749 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16916492264743 execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:33:53,751 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-16916492264743 in 1.1960 sec 2023-08-10T06:33:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-08-10T06:33:54,666 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns4:test-16916492264743, procId: 22 completed 2023-08-10T06:33:54,666 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-16916492264743 get assigned. Timeout = 60000ms 2023-08-10T06:33:54,667 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-08-10T06:33:54,671 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-16916492264743 assigned to meta. Checking AM states. 2023-08-10T06:33:54,671 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-08-10T06:33:54,671 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-16916492264743 assigned. 2023-08-10T06:33:54,671 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4f75312a to 127.0.0.1:59679 2023-08-10T06:33:54,671 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:33:54,713 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] hbase.ResourceChecker(147): before: backup.TestBackupMultipleDeletes#testBackupMultipleDeletes Thread=582, OpenFileDescriptor=660, MaxFileDescriptor=60000, SystemLoadAverage=282, ProcessCount=171, AvailableMemoryMB=2070 2023-08-10T06:33:54,713 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] hbase.ResourceChecker(130): Thread=582 is superior to 500 2023-08-10T06:33:54,715 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] backup.TestBackupMultipleDeletes(60): create full backup image for all tables 2023-08-10T06:33:54,715 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x22204ed5 to 127.0.0.1:59679 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:33:54,720 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f5bf24f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:33:54,729 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:33:54,731 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:58686, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:33:54,743 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:33:54,744 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:45120, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:33:54,754 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'backup'} 2023-08-10T06:33:54,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure2.ProcedureExecutor(1033): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-08-10T06:33:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=25 2023-08-10T06:33:54,770 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:33:54,775 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=25, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 20 msec 2023-08-10T06:33:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=25 2023-08-10T06:33:54,875 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:33:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure2.ProcedureExecutor(1033): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-08-10T06:33:54,879 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:33:54,879 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:33:54,880 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2023-08-10T06:33:54,881 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:33:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-08-10T06:33:54,918 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16916492264741 2023-08-10T06:33:54,918 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-16916492264741 Metrics about Tables on a single HBase RegionServer 2023-08-10T06:33:54,919 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16916492264742 2023-08-10T06:33:54,919 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-16916492264742 Metrics about Tables on a single HBase RegionServer 2023-08-10T06:33:54,920 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16916492264743 2023-08-10T06:33:54,920 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-16916492264743 Metrics about Tables on a single HBase RegionServer 2023-08-10T06:33:54,920 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1691649226474 2023-08-10T06:33:54,920 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1691649226474 Metrics about Tables on a single HBase RegionServer 2023-08-10T06:33:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-08-10T06:33:55,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-08-10T06:33:55,298 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 60ed3292def93d2ecd5862b25318ed47, NAME => 'backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f 2023-08-10T06:33:55,309 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:33:55,309 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing 60ed3292def93d2ecd5862b25318ed47, disabling compactions & flushes 2023-08-10T06:33:55,309 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. 2023-08-10T06:33:55,310 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. 2023-08-10T06:33:55,310 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. after waiting 0 ms 2023-08-10T06:33:55,310 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. 2023-08-10T06:33:55,310 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. 2023-08-10T06:33:55,310 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for 60ed3292def93d2ecd5862b25318ed47: 2023-08-10T06:33:55,312 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:33:55,312 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1691649235312"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691649235312"}]},"ts":"1691649235312"} 2023-08-10T06:33:55,314 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:33:55,315 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:33:55,315 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649235315"}]},"ts":"1691649235315"} 2023-08-10T06:33:55,318 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-08-10T06:33:55,322 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=60ed3292def93d2ecd5862b25318ed47, ASSIGN}] 2023-08-10T06:33:55,324 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=60ed3292def93d2ecd5862b25318ed47, ASSIGN 2023-08-10T06:33:55,325 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=60ed3292def93d2ecd5862b25318ed47, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44853,1691649217899; forceNewPlan=false, retain=false 2023-08-10T06:33:55,476 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=60ed3292def93d2ecd5862b25318ed47, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:33:55,479 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; OpenRegionProcedure 60ed3292def93d2ecd5862b25318ed47, server=jenkins-hbase3.apache.org,44853,1691649217899}] 2023-08-10T06:33:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-08-10T06:33:55,636 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(130): Open backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. 2023-08-10T06:33:55,636 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7254): Opening region: {ENCODED => 60ed3292def93d2ecd5862b25318ed47, NAME => 'backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:33:55,637 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:33:55,637 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:33:55,637 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(888): Instantiated backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:33:55,637 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7296): checking encryption for 60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:33:55,637 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7299): checking classloading for 60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:33:55,639 INFO [StoreOpener-60ed3292def93d2ecd5862b25318ed47-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:33:55,642 INFO [StoreOpener-60ed3292def93d2ecd5862b25318ed47-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 60ed3292def93d2ecd5862b25318ed47 columnFamilyName meta 2023-08-10T06:33:55,642 DEBUG [StoreOpener-60ed3292def93d2ecd5862b25318ed47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:33:55,643 INFO [StoreOpener-60ed3292def93d2ecd5862b25318ed47-1 {}] regionserver.HStore(324): Store=60ed3292def93d2ecd5862b25318ed47/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:33:55,643 INFO [StoreOpener-60ed3292def93d2ecd5862b25318ed47-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:33:55,644 INFO [StoreOpener-60ed3292def93d2ecd5862b25318ed47-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 60ed3292def93d2ecd5862b25318ed47 columnFamilyName session 2023-08-10T06:33:55,645 DEBUG [StoreOpener-60ed3292def93d2ecd5862b25318ed47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:33:55,645 INFO [StoreOpener-60ed3292def93d2ecd5862b25318ed47-1 {}] regionserver.HStore(324): Store=60ed3292def93d2ecd5862b25318ed47/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:33:55,646 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:33:55,647 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:33:55,648 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-08-10T06:33:55,650 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1076): writing seq id for 60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:33:55,652 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:33:55,653 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): Opened 60ed3292def93d2ecd5862b25318ed47; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10361539200, jitterRate=-0.03500646352767944}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-08-10T06:33:55,653 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(992): Region open journal for 60ed3292def93d2ecd5862b25318ed47: 2023-08-10T06:33:55,654 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47., pid=28, masterSystemTime=1691649235631 2023-08-10T06:33:55,657 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. 2023-08-10T06:33:55,657 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(158): Opened backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. 2023-08-10T06:33:55,657 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=60ed3292def93d2ecd5862b25318ed47, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:33:55,661 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=28, resume processing ppid=27 2023-08-10T06:33:55,662 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=28, ppid=27, state=SUCCESS; OpenRegionProcedure 60ed3292def93d2ecd5862b25318ed47, server=jenkins-hbase3.apache.org,44853,1691649217899 in 180 msec 2023-08-10T06:33:55,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=27, resume processing ppid=26 2023-08-10T06:33:55,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=60ed3292def93d2ecd5862b25318ed47, ASSIGN in 339 msec 2023-08-10T06:33:55,665 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:33:55,665 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649235665"}]},"ts":"1691649235665"} 2023-08-10T06:33:55,666 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-08-10T06:33:55,670 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:33:55,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=26, state=SUCCESS; CreateTableProcedure table=backup:system in 795 msec 2023-08-10T06:33:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-08-10T06:33:55,988 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: backup:system, procId: 26 completed 2023-08-10T06:33:56,001 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:33:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure2.ProcedureExecutor(1033): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-08-10T06:33:56,005 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:33:56,005 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:33:56,005 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2023-08-10T06:33:56,006 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:33:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-08-10T06:33:56,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-08-10T06:33:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-08-10T06:33:56,421 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 2468e04f6179571f82a0adee7c1e6692, NAME => 'backup:system_bulk,,1691649236001.2468e04f6179571f82a0adee7c1e6692.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f 2023-08-10T06:33:56,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-08-10T06:33:56,834 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1691649236001.2468e04f6179571f82a0adee7c1e6692.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:33:56,834 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing 2468e04f6179571f82a0adee7c1e6692, disabling compactions & flushes 2023-08-10T06:33:56,834 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1691649236001.2468e04f6179571f82a0adee7c1e6692. 2023-08-10T06:33:56,834 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1691649236001.2468e04f6179571f82a0adee7c1e6692. 2023-08-10T06:33:56,834 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1691649236001.2468e04f6179571f82a0adee7c1e6692. after waiting 0 ms 2023-08-10T06:33:56,834 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1691649236001.2468e04f6179571f82a0adee7c1e6692. 2023-08-10T06:33:56,834 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1691649236001.2468e04f6179571f82a0adee7c1e6692. 2023-08-10T06:33:56,834 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for 2468e04f6179571f82a0adee7c1e6692: 2023-08-10T06:33:56,836 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:33:56,836 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1691649236001.2468e04f6179571f82a0adee7c1e6692.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1691649236836"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691649236836"}]},"ts":"1691649236836"} 2023-08-10T06:33:56,838 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:33:56,839 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:33:56,840 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649236839"}]},"ts":"1691649236839"} 2023-08-10T06:33:56,841 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-08-10T06:33:56,846 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=2468e04f6179571f82a0adee7c1e6692, ASSIGN}] 2023-08-10T06:33:56,848 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=2468e04f6179571f82a0adee7c1e6692, ASSIGN 2023-08-10T06:33:56,849 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=2468e04f6179571f82a0adee7c1e6692, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44853,1691649217899; forceNewPlan=false, retain=false 2023-08-10T06:33:57,001 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=2468e04f6179571f82a0adee7c1e6692, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:33:57,003 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE; OpenRegionProcedure 2468e04f6179571f82a0adee7c1e6692, server=jenkins-hbase3.apache.org,44853,1691649217899}] 2023-08-10T06:33:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-08-10T06:33:57,159 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1691649236001.2468e04f6179571f82a0adee7c1e6692. 2023-08-10T06:33:57,160 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7254): Opening region: {ENCODED => 2468e04f6179571f82a0adee7c1e6692, NAME => 'backup:system_bulk,,1691649236001.2468e04f6179571f82a0adee7c1e6692.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:33:57,160 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:33:57,160 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk 2468e04f6179571f82a0adee7c1e6692 2023-08-10T06:33:57,160 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1691649236001.2468e04f6179571f82a0adee7c1e6692.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:33:57,160 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7296): checking encryption for 2468e04f6179571f82a0adee7c1e6692 2023-08-10T06:33:57,160 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7299): checking classloading for 2468e04f6179571f82a0adee7c1e6692 2023-08-10T06:33:57,162 INFO [StoreOpener-2468e04f6179571f82a0adee7c1e6692-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 2468e04f6179571f82a0adee7c1e6692 2023-08-10T06:33:57,164 INFO [StoreOpener-2468e04f6179571f82a0adee7c1e6692-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2468e04f6179571f82a0adee7c1e6692 columnFamilyName meta 2023-08-10T06:33:57,164 DEBUG [StoreOpener-2468e04f6179571f82a0adee7c1e6692-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:33:57,164 INFO [StoreOpener-2468e04f6179571f82a0adee7c1e6692-1 {}] regionserver.HStore(324): Store=2468e04f6179571f82a0adee7c1e6692/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:33:57,165 INFO [StoreOpener-2468e04f6179571f82a0adee7c1e6692-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 2468e04f6179571f82a0adee7c1e6692 2023-08-10T06:33:57,166 INFO [StoreOpener-2468e04f6179571f82a0adee7c1e6692-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2468e04f6179571f82a0adee7c1e6692 columnFamilyName session 2023-08-10T06:33:57,166 DEBUG [StoreOpener-2468e04f6179571f82a0adee7c1e6692-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:33:57,167 INFO [StoreOpener-2468e04f6179571f82a0adee7c1e6692-1 {}] regionserver.HStore(324): Store=2468e04f6179571f82a0adee7c1e6692/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:33:57,168 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system_bulk/2468e04f6179571f82a0adee7c1e6692 2023-08-10T06:33:57,168 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system_bulk/2468e04f6179571f82a0adee7c1e6692 2023-08-10T06:33:57,170 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-08-10T06:33:57,172 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1076): writing seq id for 2468e04f6179571f82a0adee7c1e6692 2023-08-10T06:33:57,175 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system_bulk/2468e04f6179571f82a0adee7c1e6692/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:33:57,176 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): Opened 2468e04f6179571f82a0adee7c1e6692; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11853993280, jitterRate=0.10398915410041809}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-08-10T06:33:57,176 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(992): Region open journal for 2468e04f6179571f82a0adee7c1e6692: 2023-08-10T06:33:57,177 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system_bulk,,1691649236001.2468e04f6179571f82a0adee7c1e6692., pid=31, masterSystemTime=1691649237155 2023-08-10T06:33:57,180 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system_bulk,,1691649236001.2468e04f6179571f82a0adee7c1e6692. 2023-08-10T06:33:57,180 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1691649236001.2468e04f6179571f82a0adee7c1e6692. 2023-08-10T06:33:57,180 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=2468e04f6179571f82a0adee7c1e6692, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:33:57,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=31, resume processing ppid=30 2023-08-10T06:33:57,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=31, ppid=30, state=SUCCESS; OpenRegionProcedure 2468e04f6179571f82a0adee7c1e6692, server=jenkins-hbase3.apache.org,44853,1691649217899 in 179 msec 2023-08-10T06:33:57,187 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=30, resume processing ppid=29 2023-08-10T06:33:57,187 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=30, ppid=29, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=2468e04f6179571f82a0adee7c1e6692, ASSIGN in 338 msec 2023-08-10T06:33:57,188 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:33:57,188 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649237188"}]},"ts":"1691649237188"} 2023-08-10T06:33:57,190 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-08-10T06:33:57,192 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:33:57,194 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=29, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 1.1900 sec 2023-08-10T06:33:58,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-08-10T06:33:58,114 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: backup:system_bulk, procId: 29 completed 2023-08-10T06:33:58,130 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:33:58,143 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-08-10T06:33:58,143 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-16916492264743' 2023-08-10T06:33:58,144 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-08-10T06:33:58,178 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:33:58,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649238179 (current time:1691649238179). 2023-08-10T06:33:58,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:33:58,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:33:58,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6f4b4983 to 127.0.0.1:59679 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:33:58,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1db018, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:33:58,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:33:58,192 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:58688, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:33:58,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6f4b4983 to 127.0.0.1:59679 2023-08-10T06:33:58,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:33:58,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:33:58,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure2.ProcedureExecutor(1033): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:33:58,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-08-10T06:33:58,210 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:33:58,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-08-10T06:33:58,216 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:33:58,219 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:33:58,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-08-10T06:33:58,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-08-10T06:33:58,635 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:33:58,638 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47}] 2023-08-10T06:33:58,648 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:33:58,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44853 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2023-08-10T06:33:58,810 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. 2023-08-10T06:33:58,812 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2819): Flushing 60ed3292def93d2ecd5862b25318ed47 2/2 column families, dataSize=45 B heapSize=632 B 2023-08-10T06:33:58,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-08-10T06:33:58,903 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/session/6743ea07cad64eb59ec77e7feafa1733 is 49, key is activesession:/session:c/1691649238161/Put/seqid=0 2023-08-10T06:33:59,222 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:33:59,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-08-10T06:33:59,323 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/session/6743ea07cad64eb59ec77e7feafa1733 2023-08-10T06:33:59,400 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6743ea07cad64eb59ec77e7feafa1733 2023-08-10T06:33:59,404 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/session/6743ea07cad64eb59ec77e7feafa1733 as hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/6743ea07cad64eb59ec77e7feafa1733 2023-08-10T06:33:59,415 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/6743ea07cad64eb59ec77e7feafa1733, entries=1, sequenceid=5, filesize=4.9 K 2023-08-10T06:33:59,420 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for 60ed3292def93d2ecd5862b25318ed47 in 608ms, sequenceid=5, compaction requested=false 2023-08-10T06:33:59,423 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2520): Flush status journal for 60ed3292def93d2ecd5862b25318ed47: 2023-08-10T06:33:59,423 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. for snapshot_backup_system completed. 2023-08-10T06:33:59,424 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:33:59,434 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:33:59,438 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-08-10T06:33:59,439 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/6743ea07cad64eb59ec77e7feafa1733] hfiles 2023-08-10T06:33:59,439 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/6743ea07cad64eb59ec77e7feafa1733 for snapshot=snapshot_backup_system 2023-08-10T06:33:59,459 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. 2023-08-10T06:33:59,463 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=33 2023-08-10T06:33:59,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.HMaster(3994): Remote procedure done, pid=33 2023-08-10T06:33:59,467 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:33:59,469 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:33:59,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=33, resume processing ppid=32 2023-08-10T06:33:59,473 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:33:59,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=33, ppid=32, state=SUCCESS; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47 in 832 msec 2023-08-10T06:33:59,476 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:33:59,482 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:33:59,482 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:33:59,486 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:33:59,914 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:33:59,930 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:33:59,931 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:34:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-08-10T06:34:00,377 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:34:00,377 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-08-10T06:34:00,379 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=32, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.1750 sec 2023-08-10T06:34:02,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-08-10T06:34:02,327 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 32 completed 2023-08-10T06:34:02,327 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(115): Backup backup_1691649234723 started at 1691649242327. 2023-08-10T06:34:02,350 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(119): Backup session backup_1691649234723 has been started. 2023-08-10T06:34:02,355 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-08-10T06:34:02,370 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-08-10T06:34:02,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-08-10T06:34:02,376 INFO [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-08-10T06:34:02,376 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-08-10T06:34:02,377 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-08-10T06:34:02,377 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-08-10T06:34:02,378 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-08-10T06:34:02,378 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:34:02,380 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-08-10T06:34:02,380 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:02,380 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:34:02,380 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:34:02,380 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:02,380 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-08-10T06:34:02,380 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-08-10T06:34:02,381 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-08-10T06:34:02,381 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-08-10T06:34:02,381 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:34:02,381 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-08-10T06:34:02,383 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-08-10T06:34:02,386 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-08-10T06:34:02,386 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-08-10T06:34:02,386 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-08-10T06:34:02,387 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-08-10T06:34:02,387 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-08-10T06:34:02,387 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,44853,1691649217899' joining acquired barrier for procedure (rolllog) in zk 2023-08-10T06:34:02,389 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:02,389 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:02,389 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:02,389 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:34:02,390 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:34:02,390 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-08-10T06:34:02,390 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-08-10T06:34:02,390 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:34:02,391 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:34:02,391 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:34:02,391 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:02,392 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:34:02,392 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,44853,1691649217899' joining acquired barrier for procedure 'rolllog' on coordinator 2023-08-10T06:34:02,392 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@4929addd[Count = 0] remaining members to acquire global barrier 2023-08-10T06:34:02,392 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-08-10T06:34:02,392 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:02,395 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:02,395 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:02,395 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:02,395 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-08-10T06:34:02,395 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:02,395 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-08-10T06:34:02,396 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-08-10T06:34:02,396 DEBUG [rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:02,397 INFO [rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1691649219826 highest: 1691649219826 on jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:02,397 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta:.meta(num 1691649220360) roll requested 2023-08-10T06:34:02,409 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649242397.meta, exclude list is [], retry=0 2023-08-10T06:34:02,414 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK] 2023-08-10T06:34:02,418 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649220360.meta with entries=36, filesize=11.13 KB; new WAL /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649242397.meta 2023-08-10T06:34:02,419 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK]] 2023-08-10T06:34:02,419 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649220360.meta is not closed yet, will try archiving it next time 2023-08-10T06:34:02,419 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44853%2C1691649217899:(num 1691649219826) roll requested 2023-08-10T06:34:02,433 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649242419, exclude list is [], retry=0 2023-08-10T06:34:02,437 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK] 2023-08-10T06:34:02,437 DEBUG [rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-08-10T06:34:02,437 INFO [rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1691649242419 on jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:02,438 DEBUG [rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:34:02,440 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:46466, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-08-10T06:34:02,444 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649219826 with entries=19, filesize=4.02 KB; new WAL /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649242419 2023-08-10T06:34:02,446 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK]] 2023-08-10T06:34:02,446 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649219826 is not closed yet, will try archiving it next time 2023-08-10T06:34:02,472 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-08-10T06:34:02,472 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,44853,1691649217899' in zk 2023-08-10T06:34:02,475 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:02,475 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-08-10T06:34:02,475 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:34:02,475 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:02,475 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-08-10T06:34:02,477 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:34:02,477 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:34:02,477 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:34:02,477 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:34:02,478 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:34:02,478 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:02,479 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:34:02,479 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:34:02,479 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:02,480 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,44853,1691649217899': 2023-08-10T06:34:02,480 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,44853,1691649217899' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-08-10T06:34:02,480 INFO [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-08-10T06:34:02,480 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-08-10T06:34:02,480 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-08-10T06:34:02,480 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-08-10T06:34:02,480 INFO [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-08-10T06:34:02,483 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:34:02,483 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:34:02,483 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-08-10T06:34:02,483 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-08-10T06:34:02,483 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-08-10T06:34:02,484 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:34:02,484 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:34:02,484 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:34:02,484 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:34:02,484 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:34:02,484 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:34:02,484 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:02,484 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:34:02,485 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:34:02,485 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:34:02,486 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:34:02,490 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:02,490 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:02,490 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:34:02,491 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:34:02,491 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:02,500 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:02,501 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-08-10T06:34:02,501 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-08-10T06:34:02,501 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-08-10T06:34:02,501 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:02,501 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-08-10T06:34:02,501 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:34:02,501 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:02,501 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:34:02,501 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:02,501 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:34:02,501 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:34:02,501 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-08-10T06:34:02,501 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-08-10T06:34:02,501 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:34:02,501 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:34:02,505 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin(2751): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-08-10T06:34:02,505 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin(2759): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-08-10T06:34:02,605 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin(2765): Getting current status of procedure from master... 2023-08-10T06:34:02,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-08-10T06:34:02,616 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1691649242615_default_test-1691649226474 table=test-1691649226474 type=FLUSH ttl=0 } 2023-08-10T06:34:02,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649242616 (current time:1691649242616). 2023-08-10T06:34:02,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:34:02,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1691649242615_default_test-1691649226474 VERSION not specified, setting to 2 2023-08-10T06:34:02,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3811b866 to 127.0.0.1:59679 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:34:02,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7563917d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:34:02,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:34:02,626 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:44274, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:34:02,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3811b866 to 127.0.0.1:59679 2023-08-10T06:34:02,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:34:02,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:34:02,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure2.ProcedureExecutor(1033): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691649242615_default_test-1691649226474 table=test-1691649226474 type=FLUSH ttl=0 } 2023-08-10T06:34:02,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1691649242615_default_test-1691649226474 table=test-1691649226474 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-08-10T06:34:02,631 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691649242615_default_test-1691649226474 table=test-1691649226474 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:34:02,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-08-10T06:34:02,632 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691649242615_default_test-1691649226474 table=test-1691649226474 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:34:02,634 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691649242615_default_test-1691649226474 table=test-1691649226474 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:34:02,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-08-10T06:34:02,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-08-10T06:34:03,050 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691649242615_default_test-1691649226474 table=test-1691649226474 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:34:03,050 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 3e5d3d5b3b7bc4e6a979a8585e22b81f}] 2023-08-10T06:34:03,052 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 3e5d3d5b3b7bc4e6a979a8585e22b81f 2023-08-10T06:34:03,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44853 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2023-08-10T06:34:03,205 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1691649226474,,1691649226962.3e5d3d5b3b7bc4e6a979a8585e22b81f. 2023-08-10T06:34:03,206 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2819): Flushing 3e5d3d5b3b7bc4e6a979a8585e22b81f 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-08-10T06:34:03,224 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/default/test-1691649226474/3e5d3d5b3b7bc4e6a979a8585e22b81f/.tmp/f/2ff193057568456b8bd59abdcfa367df is 37, key is row10/f:q1/1691649228135/Put/seqid=0 2023-08-10T06:34:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-08-10T06:34:03,648 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/default/test-1691649226474/3e5d3d5b3b7bc4e6a979a8585e22b81f/.tmp/f/2ff193057568456b8bd59abdcfa367df 2023-08-10T06:34:03,659 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2ff193057568456b8bd59abdcfa367df 2023-08-10T06:34:03,660 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/default/test-1691649226474/3e5d3d5b3b7bc4e6a979a8585e22b81f/.tmp/f/2ff193057568456b8bd59abdcfa367df as hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/default/test-1691649226474/3e5d3d5b3b7bc4e6a979a8585e22b81f/f/2ff193057568456b8bd59abdcfa367df 2023-08-10T06:34:03,667 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/default/test-1691649226474/3e5d3d5b3b7bc4e6a979a8585e22b81f/f/2ff193057568456b8bd59abdcfa367df, entries=99, sequenceid=103, filesize=8.2 K 2023-08-10T06:34:03,669 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 3e5d3d5b3b7bc4e6a979a8585e22b81f in 463ms, sequenceid=103, compaction requested=false 2023-08-10T06:34:03,670 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2520): Flush status journal for 3e5d3d5b3b7bc4e6a979a8585e22b81f: 2023-08-10T06:34:03,670 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1691649226474,,1691649226962.3e5d3d5b3b7bc4e6a979a8585e22b81f. for snapshot_1691649242615_default_test-1691649226474 completed. 2023-08-10T06:34:03,671 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(240): Storing 'test-1691649226474,,1691649226962.3e5d3d5b3b7bc4e6a979a8585e22b81f.' region-info for snapshot=snapshot_1691649242615_default_test-1691649226474 2023-08-10T06:34:03,671 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:34:03,671 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/default/test-1691649226474/3e5d3d5b3b7bc4e6a979a8585e22b81f/f/2ff193057568456b8bd59abdcfa367df] hfiles 2023-08-10T06:34:03,671 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/default/test-1691649226474/3e5d3d5b3b7bc4e6a979a8585e22b81f/f/2ff193057568456b8bd59abdcfa367df for snapshot=snapshot_1691649242615_default_test-1691649226474 2023-08-10T06:34:03,686 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1691649226474,,1691649226962.3e5d3d5b3b7bc4e6a979a8585e22b81f. 2023-08-10T06:34:03,686 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=35 2023-08-10T06:34:03,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.HMaster(3994): Remote procedure done, pid=35 2023-08-10T06:34:03,687 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1691649242615_default_test-1691649226474 on region 3e5d3d5b3b7bc4e6a979a8585e22b81f 2023-08-10T06:34:03,688 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 3e5d3d5b3b7bc4e6a979a8585e22b81f 2023-08-10T06:34:03,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=35, resume processing ppid=34 2023-08-10T06:34:03,692 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691649242615_default_test-1691649226474 table=test-1691649226474 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:34:03,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=35, ppid=34, state=SUCCESS; SnapshotRegionProcedure 3e5d3d5b3b7bc4e6a979a8585e22b81f in 639 msec 2023-08-10T06:34:03,693 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691649242615_default_test-1691649226474 table=test-1691649226474 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:34:03,693 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691649242615_default_test-1691649226474 table=test-1691649226474 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:34:03,693 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1691649242615_default_test-1691649226474 2023-08-10T06:34:03,695 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/.tmp/snapshot_1691649242615_default_test-1691649226474 2023-08-10T06:34:03,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-08-10T06:34:04,135 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691649242615_default_test-1691649226474 table=test-1691649226474 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:34:04,143 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691649242615_default_test-1691649226474 table=test-1691649226474 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:34:04,144 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/.tmp/snapshot_1691649242615_default_test-1691649226474 to hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/snapshot_1691649242615_default_test-1691649226474 2023-08-10T06:34:04,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-08-10T06:34:04,918 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-08-10T06:34:04,918 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-08-10T06:34:04,919 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-08-10T06:34:04,919 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-08-10T06:34:04,979 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691649242615_default_test-1691649226474 table=test-1691649226474 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:34:04,979 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1691649242615_default_test-1691649226474 table=test-1691649226474 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-08-10T06:34:04,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=34, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691649242615_default_test-1691649226474 table=test-1691649226474 type=FLUSH ttl=0 } in 2.3510 sec 2023-08-10T06:34:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-08-10T06:34:06,743 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: default:test-1691649226474, procId: 34 completed 2023-08-10T06:34:06,744 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1691649246743_ns2_test-16916492264741 table=ns2:test-16916492264741 type=FLUSH ttl=0 } 2023-08-10T06:34:06,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649246744 (current time:1691649246744). 2023-08-10T06:34:06,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:34:06,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1691649246743_ns2_test-16916492264741 VERSION not specified, setting to 2 2023-08-10T06:34:06,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5cdbaf7c to 127.0.0.1:59679 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:34:06,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@693cfe09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:34:06,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:34:06,753 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:44288, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:34:06,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5cdbaf7c to 127.0.0.1:59679 2023-08-10T06:34:06,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:34:06,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:34:06,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure2.ProcedureExecutor(1033): Stored pid=36, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1691649246743_ns2_test-16916492264741 table=ns2:test-16916492264741 type=FLUSH ttl=0 } 2023-08-10T06:34:06,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1691649246743_ns2_test-16916492264741 table=ns2:test-16916492264741 type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-08-10T06:34:06,758 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1691649246743_ns2_test-16916492264741 table=ns2:test-16916492264741 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:34:06,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-08-10T06:34:06,760 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1691649246743_ns2_test-16916492264741 table=ns2:test-16916492264741 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:34:06,762 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1691649246743_ns2_test-16916492264741 table=ns2:test-16916492264741 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:34:06,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-08-10T06:34:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-08-10T06:34:07,177 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1691649246743_ns2_test-16916492264741 table=ns2:test-16916492264741 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:34:07,177 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 94f564ab80fb7e8519555fad2ae5463b}] 2023-08-10T06:34:07,179 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 94f564ab80fb7e8519555fad2ae5463b 2023-08-10T06:34:07,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44853 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=37 2023-08-10T06:34:07,332 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16916492264741,,1691649228221.94f564ab80fb7e8519555fad2ae5463b. 2023-08-10T06:34:07,333 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2819): Flushing 94f564ab80fb7e8519555fad2ae5463b 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-08-10T06:34:07,350 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/ns2/test-16916492264741/94f564ab80fb7e8519555fad2ae5463b/.tmp/f/db978fb0aa084e4586c17125b4193ef8 is 37, key is row10/f:q1/1691649230352/Put/seqid=0 2023-08-10T06:34:07,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-08-10T06:34:07,758 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/ns2/test-16916492264741/94f564ab80fb7e8519555fad2ae5463b/.tmp/f/db978fb0aa084e4586c17125b4193ef8 2023-08-10T06:34:07,767 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: db978fb0aa084e4586c17125b4193ef8 2023-08-10T06:34:07,768 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/ns2/test-16916492264741/94f564ab80fb7e8519555fad2ae5463b/.tmp/f/db978fb0aa084e4586c17125b4193ef8 as hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/ns2/test-16916492264741/94f564ab80fb7e8519555fad2ae5463b/f/db978fb0aa084e4586c17125b4193ef8 2023-08-10T06:34:07,777 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/ns2/test-16916492264741/94f564ab80fb7e8519555fad2ae5463b/f/db978fb0aa084e4586c17125b4193ef8, entries=99, sequenceid=103, filesize=8.2 K 2023-08-10T06:34:07,778 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 94f564ab80fb7e8519555fad2ae5463b in 445ms, sequenceid=103, compaction requested=false 2023-08-10T06:34:07,779 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2520): Flush status journal for 94f564ab80fb7e8519555fad2ae5463b: 2023-08-10T06:34:07,779 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16916492264741,,1691649228221.94f564ab80fb7e8519555fad2ae5463b. for snapshot_1691649246743_ns2_test-16916492264741 completed. 2023-08-10T06:34:07,779 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16916492264741,,1691649228221.94f564ab80fb7e8519555fad2ae5463b.' region-info for snapshot=snapshot_1691649246743_ns2_test-16916492264741 2023-08-10T06:34:07,779 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:34:07,779 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/ns2/test-16916492264741/94f564ab80fb7e8519555fad2ae5463b/f/db978fb0aa084e4586c17125b4193ef8] hfiles 2023-08-10T06:34:07,780 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/ns2/test-16916492264741/94f564ab80fb7e8519555fad2ae5463b/f/db978fb0aa084e4586c17125b4193ef8 for snapshot=snapshot_1691649246743_ns2_test-16916492264741 2023-08-10T06:34:07,791 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16916492264741,,1691649228221.94f564ab80fb7e8519555fad2ae5463b. 2023-08-10T06:34:07,792 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=37 2023-08-10T06:34:07,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.HMaster(3994): Remote procedure done, pid=37 2023-08-10T06:34:07,792 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1691649246743_ns2_test-16916492264741 on region 94f564ab80fb7e8519555fad2ae5463b 2023-08-10T06:34:07,793 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 94f564ab80fb7e8519555fad2ae5463b 2023-08-10T06:34:07,797 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=37, resume processing ppid=36 2023-08-10T06:34:07,797 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1691649246743_ns2_test-16916492264741 table=ns2:test-16916492264741 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:34:07,797 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=37, ppid=36, state=SUCCESS; SnapshotRegionProcedure 94f564ab80fb7e8519555fad2ae5463b in 617 msec 2023-08-10T06:34:07,797 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1691649246743_ns2_test-16916492264741 table=ns2:test-16916492264741 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:34:07,798 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1691649246743_ns2_test-16916492264741 table=ns2:test-16916492264741 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:34:07,798 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1691649246743_ns2_test-16916492264741 2023-08-10T06:34:07,799 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/.tmp/snapshot_1691649246743_ns2_test-16916492264741 2023-08-10T06:34:07,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-08-10T06:34:08,217 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1691649246743_ns2_test-16916492264741 table=ns2:test-16916492264741 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:34:08,225 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1691649246743_ns2_test-16916492264741 table=ns2:test-16916492264741 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:34:08,226 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/.tmp/snapshot_1691649246743_ns2_test-16916492264741 to hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/snapshot_1691649246743_ns2_test-16916492264741 2023-08-10T06:34:08,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-08-10T06:34:09,051 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1691649246743_ns2_test-16916492264741 table=ns2:test-16916492264741 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:34:09,051 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1691649246743_ns2_test-16916492264741 table=ns2:test-16916492264741 type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-08-10T06:34:09,052 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=36, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1691649246743_ns2_test-16916492264741 table=ns2:test-16916492264741 type=FLUSH ttl=0 } in 2.2960 sec 2023-08-10T06:34:10,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-08-10T06:34:10,868 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: ns2:test-16916492264741, procId: 36 completed 2023-08-10T06:34:10,868 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1691649234723 2023-08-10T06:34:10,868 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-08-10T06:34:10,871 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-08-10T06:34:10,873 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1691649234723_ns2:test-16916492264741 2023-08-10T06:34:10,873 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1691649246743_ns2_test-16916492264741 to hdfs://localhost:41489/backupUT/backup_1691649234723/ns2/test-16916492264741/ 2023-08-10T06:34:10,883 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-08-10T06:34:10,915 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:41489, inputRoot=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f 2023-08-10T06:34:10,915 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37947,1691649214739_448390436_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:41489/backupUT/backup_1691649234723/ns2/test-16916492264741, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:41489/backupUT/backup_1691649234723/ns2/test-16916492264741/.hbase-snapshot/.tmp/snapshot_1691649246743_ns2_test-16916492264741 2023-08-10T06:34:10,915 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:41489, inputRoot=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f, snapshotDir=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/snapshot_1691649246743_ns2_test-16916492264741. 2023-08-10T06:34:10,928 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/snapshot_1691649246743_ns2_test-16916492264741 to hdfs://localhost:41489/backupUT/backup_1691649234723/ns2/test-16916492264741/.hbase-snapshot/.tmp/snapshot_1691649246743_ns2_test-16916492264741 2023-08-10T06:34:11,373 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:11,373 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:11,374 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:11,374 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:12,429 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/hadoop-3995973177829956964.jar 2023-08-10T06:34:12,429 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:12,430 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:12,430 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:12,431 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:12,431 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:12,432 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:12,432 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:12,433 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:12,434 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:12,435 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:34:12,436 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:34:12,437 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:34:12,437 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:34:12,438 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:34:12,438 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:34:12,439 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:34:12,439 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:34:12,439 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:34:12,440 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:34:12,440 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:34:12,441 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:34:12,445 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:34:12,445 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:34:12,446 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:34:12,446 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:34:12,446 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:34:12,447 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:34:12,448 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:34:12,509 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-08-10T06:34:12,536 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1691649246743_ns2_test-16916492264741' hfile list 2023-08-10T06:34:12,548 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-08-10T06:34:12,780 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/mapreduce.cluster.local.dir/1691649252729/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:34:12,780 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/mapreduce.cluster.local.dir/1691649252729/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:34:12,952 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-08-10T06:34:12,981 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/ns2/test-16916492264741/94f564ab80fb7e8519555fad2ae5463b/f/db978fb0aa084e4586c17125b4193ef8 output=hdfs://localhost:41489/backupUT/backup_1691649234723/ns2/test-16916492264741/archive/data/ns2/test-16916492264741/94f564ab80fb7e8519555fad2ae5463b/f/db978fb0aa084e4586c17125b4193ef8 2023-08-10T06:34:12,981 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 4.004M/sec 2023-08-10T06:34:13,846 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-08-10T06:34:13,847 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-08-10T06:34:13,856 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1691649246743_ns2_test-16916492264741 2023-08-10T06:34:13,856 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1691649246743_ns2_test-16916492264741 finished. 2023-08-10T06:34:13,856 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1691649234723_test-1691649226474 2023-08-10T06:34:13,857 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1691649242615_default_test-1691649226474 to hdfs://localhost:41489/backupUT/backup_1691649234723/default/test-1691649226474/ 2023-08-10T06:34:13,857 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-08-10T06:34:13,899 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:41489, inputRoot=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f 2023-08-10T06:34:13,899 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37947,1691649214739_-1882914373_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:41489/backupUT/backup_1691649234723/default/test-1691649226474, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:41489/backupUT/backup_1691649234723/default/test-1691649226474/.hbase-snapshot/.tmp/snapshot_1691649242615_default_test-1691649226474 2023-08-10T06:34:13,899 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:41489, inputRoot=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f, snapshotDir=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/snapshot_1691649242615_default_test-1691649226474. 2023-08-10T06:34:13,911 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/snapshot_1691649242615_default_test-1691649226474 to hdfs://localhost:41489/backupUT/backup_1691649234723/default/test-1691649226474/.hbase-snapshot/.tmp/snapshot_1691649242615_default_test-1691649226474 2023-08-10T06:34:14,245 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-08-10T06:34:14,248 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:34918, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-08-10T06:34:14,338 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:14,339 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:14,340 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:14,340 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:15,431 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/hadoop-845620648116122635.jar 2023-08-10T06:34:15,432 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:15,432 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:15,432 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:15,433 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:15,433 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:15,434 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:15,434 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:15,434 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:15,435 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:15,436 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:34:15,436 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:34:15,437 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:34:15,437 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:34:15,438 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:34:15,438 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:34:15,439 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:34:15,439 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:34:15,440 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:34:15,440 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:34:15,441 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:34:15,441 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:34:15,442 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:34:15,442 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:34:15,443 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:34:15,443 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:34:15,444 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:34:15,444 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:34:15,445 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:34:15,473 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-08-10T06:34:15,491 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1691649242615_default_test-1691649226474' hfile list 2023-08-10T06:34:15,494 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-08-10T06:34:15,572 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/mapreduce.cluster.local.dir/1691649255535/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:34:15,572 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/mapreduce.cluster.local.dir/1691649255535/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:34:15,636 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-08-10T06:34:15,668 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/default/test-1691649226474/3e5d3d5b3b7bc4e6a979a8585e22b81f/f/2ff193057568456b8bd59abdcfa367df output=hdfs://localhost:41489/backupUT/backup_1691649234723/default/test-1691649226474/archive/data/default/test-1691649226474/3e5d3d5b3b7bc4e6a979a8585e22b81f/f/2ff193057568456b8bd59abdcfa367df 2023-08-10T06:34:15,668 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 4.004M/sec 2023-08-10T06:34:16,615 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-08-10T06:34:16,616 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-08-10T06:34:16,624 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1691649242615_default_test-1691649226474 2023-08-10T06:34:16,625 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1691649242615_default_test-1691649226474 finished. 2023-08-10T06:34:16,625 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(1014): ns2:test-16916492264741 2023-08-10T06:34:16,625 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(1014): test-1691649226474 2023-08-10T06:34:16,721 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649234723 2023-08-10T06:34:16,721 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-08-10T06:34:17,135 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:41489/backupUT/backup_1691649234723/.backup.manifest 2023-08-10T06:34:17,135 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649234723 2023-08-10T06:34:17,135 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-08-10T06:34:17,542 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:41489/backupUT/backup_1691649234723/.backup.manifest 2023-08-10T06:34:17,542 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(358): Backup backup_1691649234723 finished: type=FULL,tablelist=ns2:test-16916492264741;test-1691649226474,targetRootDir=hdfs://localhost:41489/backupUT,startts=1691649242327,completets=1691649256717,bytescopied=0 2023-08-10T06:34:17,542 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-08-10T06:34:17,542 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1691649246743_ns2_test-16916492264741 2023-08-10T06:34:17,553 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1691649246743_ns2_test-16916492264741" 2023-08-10T06:34:17,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1691649246743_ns2_test-16916492264741 2023-08-10T06:34:17,559 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1691649246743_ns2_test-16916492264741 for backup backup_1691649234723 succeeded. 2023-08-10T06:34:17,559 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1691649242615_default_test-1691649226474 2023-08-10T06:34:17,560 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1691649242615_default_test-1691649226474" 2023-08-10T06:34:17,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1691649242615_default_test-1691649226474 2023-08-10T06:34:17,564 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1691649242615_default_test-1691649226474 for backup backup_1691649234723 succeeded. 2023-08-10T06:34:17,565 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:34:17,580 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:34:17,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:34:17,584 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:34:17,592 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:34:17,609 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(376): Backup backup_1691649234723 completed. 2023-08-10T06:34:17,791 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:34:17,795 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:34:17,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649257795 (current time:1691649257795). 2023-08-10T06:34:17,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:34:17,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:34:17,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x16818f18 to 127.0.0.1:59679 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:34:17,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65a19be3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:34:17,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:34:17,804 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:55630, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:34:17,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x16818f18 to 127.0.0.1:59679 2023-08-10T06:34:17,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:34:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:34:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure2.ProcedureExecutor(1033): Stored pid=38, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:34:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 38 2023-08-10T06:34:17,808 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:34:17,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-08-10T06:34:17,809 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:34:17,811 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:34:17,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-08-10T06:34:18,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-08-10T06:34:18,218 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:34:18,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47}] 2023-08-10T06:34:18,221 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=39, ppid=38, state=RUNNABLE; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:34:18,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44853 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=39 2023-08-10T06:34:18,373 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. 2023-08-10T06:34:18,374 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HRegion(2819): Flushing 60ed3292def93d2ecd5862b25318ed47 2/2 column families, dataSize=1.51 KB heapSize=2.80 KB 2023-08-10T06:34:18,388 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/meta/ee5a7bc3977042ada997c293856052bb is 171, key is trslm:hdfs://localhost:41489/backupUT\x00ns2:test-16916492264741/meta:log-roll-map/1691649256706/Put/seqid=0 2023-08-10T06:34:18,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-08-10T06:34:18,794 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=785 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/meta/ee5a7bc3977042ada997c293856052bb 2023-08-10T06:34:18,801 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: ee5a7bc3977042ada997c293856052bb 2023-08-10T06:34:18,804 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/session/0eca886709dc4bc28ce13c5c7fbf8954 is 459, key is session:backup_1691649234723/session:context/1691649257591/Put/seqid=0 2023-08-10T06:34:18,809 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=758 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/session/0eca886709dc4bc28ce13c5c7fbf8954 2023-08-10T06:34:18,816 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 0eca886709dc4bc28ce13c5c7fbf8954 2023-08-10T06:34:18,818 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/meta/ee5a7bc3977042ada997c293856052bb as hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/ee5a7bc3977042ada997c293856052bb 2023-08-10T06:34:18,825 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/ee5a7bc3977042ada997c293856052bb, entries=6, sequenceid=17, filesize=5.8 K 2023-08-10T06:34:18,826 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/session/0eca886709dc4bc28ce13c5c7fbf8954 as hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/0eca886709dc4bc28ce13c5c7fbf8954 2023-08-10T06:34:18,838 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/0eca886709dc4bc28ce13c5c7fbf8954, entries=2, sequenceid=17, filesize=5.4 K 2023-08-10T06:34:18,839 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HRegion(3022): Finished flush of dataSize ~1.51 KB/1543, heapSize ~2.77 KB/2840, currentSize=0 B/0 for 60ed3292def93d2ecd5862b25318ed47 in 466ms, sequenceid=17, compaction requested=false 2023-08-10T06:34:18,839 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HRegion(2520): Flush status journal for 60ed3292def93d2ecd5862b25318ed47: 2023-08-10T06:34:18,839 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. for snapshot_backup_system completed. 2023-08-10T06:34:18,839 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:34:18,840 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:34:18,840 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/ee5a7bc3977042ada997c293856052bb] hfiles 2023-08-10T06:34:18,840 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/ee5a7bc3977042ada997c293856052bb for snapshot=snapshot_backup_system 2023-08-10T06:34:18,840 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/6743ea07cad64eb59ec77e7feafa1733, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/0eca886709dc4bc28ce13c5c7fbf8954] hfiles 2023-08-10T06:34:18,840 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/6743ea07cad64eb59ec77e7feafa1733 for snapshot=snapshot_backup_system 2023-08-10T06:34:18,840 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/0eca886709dc4bc28ce13c5c7fbf8954 for snapshot=snapshot_backup_system 2023-08-10T06:34:18,845 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. 2023-08-10T06:34:18,846 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=39 2023-08-10T06:34:18,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.HMaster(3994): Remote procedure done, pid=39 2023-08-10T06:34:18,846 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:34:18,847 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=39, ppid=38, state=RUNNABLE; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:34:18,850 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=39, resume processing ppid=38 2023-08-10T06:34:18,851 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:34:18,851 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=39, ppid=38, state=SUCCESS; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47 in 629 msec 2023-08-10T06:34:18,852 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:34:18,852 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:34:18,852 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:34:18,853 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:34:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-08-10T06:34:19,273 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:34:19,281 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:34:19,282 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:34:19,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-08-10T06:34:20,103 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:34:20,103 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 38 2023-08-10T06:34:20,104 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=38, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2970 sec 2023-08-10T06:34:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-08-10T06:34:21,918 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 38 completed 2023-08-10T06:34:21,918 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(115): Backup backup_1691649257759 started at 1691649261918. 2023-08-10T06:34:21,920 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(119): Backup session backup_1691649257759 has been started. 2023-08-10T06:34:21,927 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalTableBackupClient(268): For incremental backup, current table set is [test-1691649226474, ns2:test-16916492264741] 2023-08-10T06:34:21,933 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(76): StartCode 1691649219826for backupID backup_1691649257759 2023-08-10T06:34:21,933 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-08-10T06:34:21,934 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-08-10T06:34:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-08-10T06:34:21,934 INFO [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-08-10T06:34:21,934 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-08-10T06:34:21,935 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-08-10T06:34:21,935 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-08-10T06:34:21,936 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-08-10T06:34:21,936 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:34:21,937 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:21,937 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-08-10T06:34:21,937 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:34:21,937 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:34:21,937 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:21,937 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-08-10T06:34:21,937 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-08-10T06:34:21,938 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-08-10T06:34:21,938 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-08-10T06:34:21,938 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:34:21,938 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-08-10T06:34:21,938 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-08-10T06:34:21,938 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-08-10T06:34:21,938 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-08-10T06:34:21,938 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-08-10T06:34:21,939 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-08-10T06:34:21,939 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-08-10T06:34:21,939 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,44853,1691649217899' joining acquired barrier for procedure (rolllog) in zk 2023-08-10T06:34:21,941 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:21,941 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:21,942 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:21,942 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:34:21,942 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:34:21,942 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-08-10T06:34:21,942 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-08-10T06:34:21,942 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:34:21,942 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:34:21,943 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:34:21,943 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:21,943 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:34:21,944 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,44853,1691649217899' joining acquired barrier for procedure 'rolllog' on coordinator 2023-08-10T06:34:21,944 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-08-10T06:34:21,944 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@43b635cc[Count = 0] remaining members to acquire global barrier 2023-08-10T06:34:21,944 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:21,945 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:21,945 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:21,945 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:21,945 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-08-10T06:34:21,945 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:21,945 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-08-10T06:34:21,945 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-08-10T06:34:21,945 DEBUG [rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:21,946 INFO [rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1691649242419 highest: 1691649242419 on jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:21,946 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta:.meta(num 1691649242397) roll requested 2023-08-10T06:34:21,957 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649261946.meta, exclude list is [], retry=0 2023-08-10T06:34:21,961 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK] 2023-08-10T06:34:21,963 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649242397.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649261946.meta 2023-08-10T06:34:21,963 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK]] 2023-08-10T06:34:21,963 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649242397.meta is not closed yet, will try archiving it next time 2023-08-10T06:34:21,963 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44853%2C1691649217899:(num 1691649242419) roll requested 2023-08-10T06:34:21,968 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649242397.meta not finished, retry = 0 2023-08-10T06:34:21,975 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649261963, exclude list is [], retry=0 2023-08-10T06:34:21,980 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK] 2023-08-10T06:34:21,981 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649242419 with entries=113, filesize=13.97 KB; new WAL /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649261963 2023-08-10T06:34:21,982 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK]] 2023-08-10T06:34:21,982 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649242419 is not closed yet, will try archiving it next time 2023-08-10T06:34:21,984 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649242419 not finished, retry = 0 2023-08-10T06:34:21,986 DEBUG [rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-08-10T06:34:21,986 INFO [rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1691649261963 on jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:21,993 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-08-10T06:34:21,993 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-08-10T06:34:21,995 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-08-10T06:34:21,995 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,44853,1691649217899' in zk 2023-08-10T06:34:21,998 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:21,998 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-08-10T06:34:21,998 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:34:21,998 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-08-10T06:34:21,999 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:21,999 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:34:21,999 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:34:21,999 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:34:22,000 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:34:22,000 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:34:22,000 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:22,001 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:34:22,001 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:34:22,001 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:22,002 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,44853,1691649217899': 2023-08-10T06:34:22,002 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,44853,1691649217899' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-08-10T06:34:22,002 INFO [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-08-10T06:34:22,002 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-08-10T06:34:22,002 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-08-10T06:34:22,002 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-08-10T06:34:22,002 INFO [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-08-10T06:34:22,004 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:34:22,004 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:34:22,004 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-08-10T06:34:22,004 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-08-10T06:34:22,004 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-08-10T06:34:22,004 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:34:22,004 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:34:22,004 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:34:22,004 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:34:22,004 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:34:22,004 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:34:22,004 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:22,005 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:34:22,005 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:34:22,005 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:34:22,006 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:34:22,006 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:22,006 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:22,006 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:34:22,007 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:34:22,007 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:22,010 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:22,010 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-08-10T06:34:22,010 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-08-10T06:34:22,010 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-08-10T06:34:22,010 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-08-10T06:34:22,010 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:34:22,010 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:22,010 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:34:22,010 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:34:22,010 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:22,010 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-08-10T06:34:22,010 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:22,010 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-08-10T06:34:22,010 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2023-08-10T06:34:22,010 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:34:22,011 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin(2751): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-08-10T06:34:22,011 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin(2759): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-08-10T06:34:22,011 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:34:22,011 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:34:22,070 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649242397.meta to hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649242397.meta 2023-08-10T06:34:22,112 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin(2765): Getting current status of procedure from master... 2023-08-10T06:34:22,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-08-10T06:34:22,115 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:44853=1691649219826} newestTimestamps: {jenkins-hbase3.apache.org:44853=1691649242419} 2023-08-10T06:34:22,117 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649220360.meta 2023-08-10T06:34:22,117 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649261946.meta 2023-08-10T06:34:22,117 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649242419 2023-08-10T06:34:22,117 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649261963 2023-08-10T06:34:22,118 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649242397.meta 2023-08-10T06:34:22,529 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(146): Attempting to copy table info for:ns2:test-16916492264741 target: hdfs://localhost:41489/backupUT/backup_1691649257759/ns2/test-16916492264741 descriptor: 'ns2:test-16916492264741', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:34:22,529 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-08-10T06:34:22,531 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(151): Starting to write region info for table ns2:test-16916492264741 2023-08-10T06:34:22,937 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(158): Finished writing region info for table ns2:test-16916492264741 2023-08-10T06:34:23,349 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(146): Attempting to copy table info for:test-1691649226474 target: hdfs://localhost:41489/backupUT/backup_1691649257759/default/test-1691649226474 descriptor: 'test-1691649226474', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:34:23,349 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-08-10T06:34:23,352 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(151): Starting to write region info for table test-1691649226474 2023-08-10T06:34:23,758 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(158): Finished writing region info for table test-1691649226474 2023-08-10T06:34:23,767 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:41489/backupUT/.tmp/backup_1691649257759 from hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649242419 2023-08-10T06:34:23,771 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x721aa05a to 127.0.0.1:59679 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:34:23,801 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34074307, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:34:23,803 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:34:23,804 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:49604, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:34:23,820 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-08-10T06:34:23,820 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1691649226474 2023-08-10T06:34:23,825 DEBUG [hconnection-0x34fddc7f-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:34:23,826 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:54668, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:34:23,829 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1691649226474: test-1691649226474; 2023-08-10T06:34:23,830 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16916492264741 2023-08-10T06:34:23,832 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16916492264741: ns2:test-16916492264741; 2023-08-10T06:34:23,832 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(694): Configuring 2 reduce partitions to match current region count for all tables 2023-08-10T06:34:23,832 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/hbase-staging/partitions_92eb6c7c-dd0e-4bf0-ba28-f088df79b45d 2023-08-10T06:34:24,271 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:24,271 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:24,272 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:24,272 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:25,273 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/hadoop-4270838717017551365.jar 2023-08-10T06:34:25,274 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:25,274 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:25,275 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:25,275 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:25,276 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:25,276 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:25,276 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:25,277 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:25,277 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:25,278 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:34:25,278 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:34:25,279 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:34:25,279 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:34:25,280 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:34:25,280 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:34:25,281 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:34:25,281 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:34:25,282 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:34:25,282 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:34:25,283 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:34:25,283 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:34:25,285 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:25,286 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:25,286 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:25,286 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:25,287 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:25,287 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:25,288 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:34:25,288 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: default:test-1691649226474,ns2:test-16916492264741 2023-08-10T06:34:25,288 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] client.ConnectionImplementation(2114): Closing master protocol: MasterService 2023-08-10T06:34:25,288 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x721aa05a to 127.0.0.1:59679 2023-08-10T06:34:25,288 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:34:25,289 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:34:26,263 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/hadoop-3097587170694956799.jar 2023-08-10T06:34:26,269 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-08-10T06:34:26,284 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649242419 for WAL files 2023-08-10T06:34:26,285 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649242419 2023-08-10T06:34:26,364 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/mapreduce.cluster.local.dir/1691649266328/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:34:26,365 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/mapreduce.cluster.local.dir/1691649266328/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:34:26,446 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649242419 for hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649242419 (-9223372036854775808:9223372036854775807) length:14316 2023-08-10T06:34:26,464 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-08-10T06:34:26,465 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-08-10T06:34:26,589 DEBUG [pool-202-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x485ec527 to 127.0.0.1:59679 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:34:26,594 DEBUG [pool-202-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1819d173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:34:26,599 DEBUG [hconnection-0x4db92087-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:34:26,600 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:54678, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:34:26,602 DEBUG [pool-202-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x485ec527 to 127.0.0.1:59679 2023-08-10T06:34:26,602 DEBUG [pool-202-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:34:26,604 DEBUG [pool-202-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-t10] 2023-08-10T06:34:26,604 DEBUG [pool-202-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-08-10T06:34:26,618 DEBUG [pool-202-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41489/backupUT/.tmp/backup_1691649257759/_temporary/0/_temporary/attempt_local1939653228_0003_r_000001_0/default/test-1691649226474/f/329f8210d74d4e31af4047b3ececbb03 is 40, key is row-t110/f:q1/1691649257642/Put/seqid=0 2023-08-10T06:34:26,621 WARN [Thread-880 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:44853] Specified favored nodes: [jenkins-hbase3.apache.org:44853] 2023-08-10T06:34:27,401 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:41489/backupUT 2023-08-10T06:34:27,401 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1691649257759 2023-08-10T06:34:27,402 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-08-10T06:34:27,417 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:41489/backupUT/.tmp/backup_1691649257759, hdfs://localhost:41489/backupUT] 2023-08-10T06:34:27,472 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-08-10T06:34:28,053 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-08-10T06:34:28,055 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1691649257759 2023-08-10T06:34:28,055 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 8797 bytes copied." 2023-08-10T06:34:28,056 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local1913456331_0004 completed: true true 2023-08-10T06:34:28,061 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=3775863 FILE: Number of bytes written=5067414 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=109861 HDFS: Number of bytes written=2325804 HDFS: Number of read operations=471 HDFS: Number of large read operations=0 HDFS: Number of write operations=208 Map-Reduce Framework Map input records=6 Map output records=0 Input split bytes=341 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1535115264 File Input Format Counters Bytes Read=1478 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=8797 Bytes Expected=8797 Files Copied=2 DIR_COPY=4 2023-08-10T06:34:28,061 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:41489/backupUT/.tmp/backup_1691649257759 to hdfs://localhost:41489/backupUT finished. 2023-08-10T06:34:28,074 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(433): deleted 0 original bulkload rows 2023-08-10T06:34:28,074 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649257759 2023-08-10T06:34:28,082 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(340): Got 1 ancestors for the current backup. 2023-08-10T06:34:28,082 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649257759 2023-08-10T06:34:28,083 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(340): Got 1 ancestors for the current backup. 2023-08-10T06:34:28,490 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:41489/backupUT/backup_1691649257759/.backup.manifest 2023-08-10T06:34:28,490 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649257759 2023-08-10T06:34:28,493 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(340): Got 1 ancestors for the current backup. 2023-08-10T06:34:28,493 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649257759 2023-08-10T06:34:28,495 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(340): Got 1 ancestors for the current backup. 2023-08-10T06:34:28,500 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:41489/backupUT/backup_1691649257759/.backup.manifest 2023-08-10T06:34:28,500 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649257759 2023-08-10T06:34:28,502 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(340): Got 1 ancestors for the current backup. 2023-08-10T06:34:28,908 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:41489/backupUT/backup_1691649257759/.backup.manifest 2023-08-10T06:34:28,908 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(358): Backup backup_1691649257759 finished: type=INCREMENTAL,tablelist=ns2:test-16916492264741;test-1691649226474,targetRootDir=hdfs://localhost:41489/backupUT,startts=1691649261918,completets=1691649268074,bytescopied=0 2023-08-10T06:34:28,908 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:34:28,915 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:34:28,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:34:28,918 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:34:28,920 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:34:28,924 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(376): Backup backup_1691649257759 completed. 2023-08-10T06:34:29,079 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:34:29,083 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:34:29,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649269083 (current time:1691649269083). 2023-08-10T06:34:29,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:34:29,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:34:29,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x44a3d64d to 127.0.0.1:59679 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:34:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c6fd701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:34:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:34:29,092 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:54680, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:34:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x44a3d64d to 127.0.0.1:59679 2023-08-10T06:34:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:34:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:34:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure2.ProcedureExecutor(1033): Stored pid=40, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:34:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 40 2023-08-10T06:34:29,095 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:34:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=40 2023-08-10T06:34:29,096 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:34:29,098 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:34:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=40 2023-08-10T06:34:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=40 2023-08-10T06:34:29,505 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:34:29,505 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47}] 2023-08-10T06:34:29,507 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=41, ppid=40, state=RUNNABLE; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:34:29,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44853 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=41 2023-08-10T06:34:29,659 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. 2023-08-10T06:34:29,659 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.HRegion(2819): Flushing 60ed3292def93d2ecd5862b25318ed47 2/2 column families, dataSize=1.37 KB heapSize=2.52 KB 2023-08-10T06:34:29,663 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/meta/d4291c086d8c4791bd00119e10f93814 is 171, key is trslm:hdfs://localhost:41489/backupUT\x00ns2:test-16916492264741/meta:log-roll-map/1691649268065/Put/seqid=0 2023-08-10T06:34:29,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=40 2023-08-10T06:34:30,069 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=531 B at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/meta/d4291c086d8c4791bd00119e10f93814 2023-08-10T06:34:30,075 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: d4291c086d8c4791bd00119e10f93814 2023-08-10T06:34:30,078 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/session/a9dfa7ceabf344a68958248aee65d7ab is 360, key is session:backup_1691649257759/session:context/1691649268919/Put/seqid=0 2023-08-10T06:34:30,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=40 2023-08-10T06:34:30,483 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=873 B at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/session/a9dfa7ceabf344a68958248aee65d7ab 2023-08-10T06:34:30,489 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a9dfa7ceabf344a68958248aee65d7ab 2023-08-10T06:34:30,490 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/meta/d4291c086d8c4791bd00119e10f93814 as hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/d4291c086d8c4791bd00119e10f93814 2023-08-10T06:34:30,496 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/d4291c086d8c4791bd00119e10f93814, entries=4, sequenceid=28, filesize=5.7 K 2023-08-10T06:34:30,497 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/session/a9dfa7ceabf344a68958248aee65d7ab as hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/a9dfa7ceabf344a68958248aee65d7ab 2023-08-10T06:34:30,503 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/a9dfa7ceabf344a68958248aee65d7ab, entries=2, sequenceid=28, filesize=5.3 K 2023-08-10T06:34:30,504 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.HRegion(3022): Finished flush of dataSize ~1.37 KB/1404, heapSize ~2.49 KB/2552, currentSize=0 B/0 for 60ed3292def93d2ecd5862b25318ed47 in 845ms, sequenceid=28, compaction requested=true 2023-08-10T06:34:30,504 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.HRegion(2520): Flush status journal for 60ed3292def93d2ecd5862b25318ed47: 2023-08-10T06:34:30,504 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. for snapshot_backup_system completed. 2023-08-10T06:34:30,504 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:34:30,504 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:34:30,504 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/ee5a7bc3977042ada997c293856052bb, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/d4291c086d8c4791bd00119e10f93814] hfiles 2023-08-10T06:34:30,504 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/ee5a7bc3977042ada997c293856052bb for snapshot=snapshot_backup_system 2023-08-10T06:34:30,504 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/d4291c086d8c4791bd00119e10f93814 for snapshot=snapshot_backup_system 2023-08-10T06:34:30,504 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/6743ea07cad64eb59ec77e7feafa1733, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/0eca886709dc4bc28ce13c5c7fbf8954, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/a9dfa7ceabf344a68958248aee65d7ab] hfiles 2023-08-10T06:34:30,504 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/6743ea07cad64eb59ec77e7feafa1733 for snapshot=snapshot_backup_system 2023-08-10T06:34:30,504 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/0eca886709dc4bc28ce13c5c7fbf8954 for snapshot=snapshot_backup_system 2023-08-10T06:34:30,504 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/a9dfa7ceabf344a68958248aee65d7ab for snapshot=snapshot_backup_system 2023-08-10T06:34:30,911 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. 2023-08-10T06:34:30,911 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=41 2023-08-10T06:34:30,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.HMaster(3994): Remote procedure done, pid=41 2023-08-10T06:34:30,911 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:34:30,913 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=41, ppid=40, state=RUNNABLE; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:34:30,915 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=41, resume processing ppid=40 2023-08-10T06:34:30,915 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:34:30,915 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=41, ppid=40, state=SUCCESS; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47 in 1.4080 sec 2023-08-10T06:34:30,916 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:34:30,916 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:34:30,916 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:34:30,917 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:34:31,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=40 2023-08-10T06:34:31,328 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:34:31,337 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:34:31,338 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:34:32,158 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:34:32,158 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 40 2023-08-10T06:34:32,159 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=40, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0650 sec 2023-08-10T06:34:33,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=40 2023-08-10T06:34:33,203 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 40 completed 2023-08-10T06:34:33,203 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(115): Backup backup_1691649269054 started at 1691649273203. 2023-08-10T06:34:33,205 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(119): Backup session backup_1691649269054 has been started. 2023-08-10T06:34:33,206 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalTableBackupClient(268): For incremental backup, current table set is [test-1691649226474, ns2:test-16916492264741] 2023-08-10T06:34:33,213 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(76): StartCode 1691649242419for backupID backup_1691649269054 2023-08-10T06:34:33,214 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-08-10T06:34:33,214 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-08-10T06:34:33,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-08-10T06:34:33,215 INFO [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-08-10T06:34:33,215 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-08-10T06:34:33,216 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-08-10T06:34:33,216 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-08-10T06:34:33,216 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-08-10T06:34:33,216 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:34:33,218 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:33,218 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-08-10T06:34:33,218 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:34:33,218 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:34:33,218 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:33,218 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-08-10T06:34:33,218 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-08-10T06:34:33,219 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-08-10T06:34:33,219 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-08-10T06:34:33,219 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:34:33,219 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-08-10T06:34:33,219 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-08-10T06:34:33,219 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-08-10T06:34:33,220 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-08-10T06:34:33,220 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-08-10T06:34:33,221 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-08-10T06:34:33,221 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-08-10T06:34:33,221 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,44853,1691649217899' joining acquired barrier for procedure (rolllog) in zk 2023-08-10T06:34:33,229 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:33,229 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:33,229 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:33,229 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:34:33,229 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:34:33,230 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-08-10T06:34:33,230 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-08-10T06:34:33,230 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:34:33,230 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:34:33,231 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:34:33,231 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:33,231 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:34:33,232 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,44853,1691649217899' joining acquired barrier for procedure 'rolllog' on coordinator 2023-08-10T06:34:33,232 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@38357975[Count = 0] remaining members to acquire global barrier 2023-08-10T06:34:33,232 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-08-10T06:34:33,232 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:33,233 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:33,233 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:33,233 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:33,233 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-08-10T06:34:33,234 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:33,234 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-08-10T06:34:33,234 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-08-10T06:34:33,234 DEBUG [rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:33,234 INFO [rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1691649261963 highest: 1691649261963 on jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:33,234 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta:.meta(num 1691649261946) roll requested 2023-08-10T06:34:33,247 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649273234.meta, exclude list is [], retry=0 2023-08-10T06:34:33,254 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK] 2023-08-10T06:34:33,257 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649261946.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649273234.meta 2023-08-10T06:34:33,257 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK]] 2023-08-10T06:34:33,257 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649261946.meta is not closed yet, will try archiving it next time 2023-08-10T06:34:33,257 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44853%2C1691649217899:(num 1691649261963) roll requested 2023-08-10T06:34:33,259 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649261946.meta not finished, retry = 0 2023-08-10T06:34:33,270 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649273257, exclude list is [], retry=0 2023-08-10T06:34:33,275 DEBUG [rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 41 2023-08-10T06:34:33,275 INFO [rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1691649273257 on jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:33,275 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK] 2023-08-10T06:34:33,278 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649261963 with entries=109, filesize=13.33 KB; new WAL /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649273257 2023-08-10T06:34:33,278 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK]] 2023-08-10T06:34:33,278 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649261963 is not closed yet, will try archiving it next time 2023-08-10T06:34:33,289 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-08-10T06:34:33,289 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,44853,1691649217899' in zk 2023-08-10T06:34:33,292 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:33,292 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-08-10T06:34:33,292 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:34:33,292 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-08-10T06:34:33,292 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:33,293 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:34:33,293 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:34:33,293 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:34:33,293 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:34:33,294 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:34:33,294 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:33,294 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:34:33,294 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:34:33,295 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:33,295 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,44853,1691649217899': 2023-08-10T06:34:33,295 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,44853,1691649217899' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-08-10T06:34:33,295 INFO [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-08-10T06:34:33,295 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-08-10T06:34:33,295 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-08-10T06:34:33,295 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-08-10T06:34:33,295 INFO [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-08-10T06:34:33,297 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:34:33,297 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:34:33,297 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-08-10T06:34:33,297 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-08-10T06:34:33,297 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-08-10T06:34:33,297 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:34:33,297 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:34:33,297 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:34:33,297 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:34:33,297 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:34:33,297 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:34:33,298 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:34:33,298 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:33,298 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:34:33,298 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:34:33,299 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:34:33,299 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:33,302 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:33,302 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:34:33,302 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:34:33,303 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:33,306 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:33,306 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-08-10T06:34:33,306 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-08-10T06:34:33,306 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-08-10T06:34:33,306 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-08-10T06:34:33,306 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:34:33,306 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:34:33,306 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-08-10T06:34:33,306 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:33,306 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-08-10T06:34:33,306 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:34:33,307 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:33,307 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:33,307 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:34:33,307 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin(2751): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-08-10T06:34:33,307 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin(2759): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-08-10T06:34:33,307 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:34:33,307 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:34:33,360 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649261946.meta to hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649261946.meta 2023-08-10T06:34:33,407 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin(2765): Getting current status of procedure from master... 2023-08-10T06:34:33,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-08-10T06:34:33,411 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:44853=1691649242419} newestTimestamps: {jenkins-hbase3.apache.org:44853=1691649261963} 2023-08-10T06:34:33,413 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649220360.meta 2023-08-10T06:34:33,413 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649273234.meta 2023-08-10T06:34:33,413 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649261963 2023-08-10T06:34:33,413 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649273257 2023-08-10T06:34:33,414 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649242397.meta 2023-08-10T06:34:33,414 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649261946.meta 2023-08-10T06:34:33,425 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(146): Attempting to copy table info for:ns2:test-16916492264741 target: hdfs://localhost:41489/backupUT/backup_1691649269054/ns2/test-16916492264741 descriptor: 'ns2:test-16916492264741', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:34:33,425 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-08-10T06:34:33,427 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(151): Starting to write region info for table ns2:test-16916492264741 2023-08-10T06:34:33,834 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(158): Finished writing region info for table ns2:test-16916492264741 2023-08-10T06:34:34,247 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(146): Attempting to copy table info for:test-1691649226474 target: hdfs://localhost:41489/backupUT/backup_1691649269054/default/test-1691649226474 descriptor: 'test-1691649226474', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:34:34,247 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-08-10T06:34:34,249 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(151): Starting to write region info for table test-1691649226474 2023-08-10T06:34:34,655 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(158): Finished writing region info for table test-1691649226474 2023-08-10T06:34:34,661 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:41489/backupUT/.tmp/backup_1691649269054 from hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649261963 2023-08-10T06:34:34,662 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x07806dbd to 127.0.0.1:59679 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:34:34,667 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dd63487, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:34:34,669 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:34:34,670 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:45702, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:34:34,672 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-08-10T06:34:34,672 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1691649226474 2023-08-10T06:34:34,677 DEBUG [hconnection-0x743eeb68-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:34:34,678 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56204, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:34:34,679 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1691649226474: test-1691649226474; 2023-08-10T06:34:34,679 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16916492264741 2023-08-10T06:34:34,682 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16916492264741: ns2:test-16916492264741; 2023-08-10T06:34:34,682 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(694): Configuring 2 reduce partitions to match current region count for all tables 2023-08-10T06:34:34,682 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/hbase-staging/partitions_4e8ec556-c303-4acd-b588-71580ddbee06 2023-08-10T06:34:35,089 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:35,090 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:35,090 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:35,091 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:36,057 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/hadoop-3582244614614113346.jar 2023-08-10T06:34:36,058 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:36,058 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:36,059 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:36,059 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:36,059 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:36,060 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:36,060 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:36,060 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:36,061 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:36,062 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:34:36,062 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:34:36,063 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:34:36,063 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:34:36,063 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:34:36,064 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:34:36,064 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:34:36,064 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:34:36,065 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:34:36,065 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:34:36,066 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:34:36,066 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:34:36,067 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:36,067 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:36,068 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:36,068 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:36,068 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:36,068 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:36,069 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:34:36,069 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: default:test-1691649226474,ns2:test-16916492264741 2023-08-10T06:34:36,069 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] client.ConnectionImplementation(2114): Closing master protocol: MasterService 2023-08-10T06:34:36,070 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x07806dbd to 127.0.0.1:59679 2023-08-10T06:34:36,070 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:34:36,071 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:34:37,032 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/hadoop-4934063843797222811.jar 2023-08-10T06:34:37,037 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-08-10T06:34:37,051 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649261963 for WAL files 2023-08-10T06:34:37,052 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649261963 2023-08-10T06:34:37,125 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/mapreduce.cluster.local.dir/1691649277089/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:34:37,125 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/mapreduce.cluster.local.dir/1691649277089/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:34:37,187 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649261963 for hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649261963 (-9223372036854775808:9223372036854775807) length:13653 2023-08-10T06:34:37,193 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-08-10T06:34:37,193 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-08-10T06:34:37,207 DEBUG [pool-215-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0cb4e23c to 127.0.0.1:59679 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:34:37,213 DEBUG [pool-215-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d6c845d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:34:37,215 DEBUG [hconnection-0x3f20a5b1-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:34:37,217 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56212, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:34:37,218 DEBUG [pool-215-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0cb4e23c to 127.0.0.1:59679 2023-08-10T06:34:37,218 DEBUG [pool-215-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:34:37,219 DEBUG [pool-215-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-t20] 2023-08-10T06:34:37,219 DEBUG [pool-215-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-08-10T06:34:37,227 DEBUG [pool-215-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41489/backupUT/.tmp/backup_1691649269054/_temporary/0/_temporary/attempt_local1318529316_0005_r_000000_0/ns2/test-16916492264741/f/67b53335a31646ac9ce32d7af4904f3e is 40, key is row-t210/f:q1/1691649268962/Put/seqid=0 2023-08-10T06:34:37,229 WARN [Thread-1013 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:44853] Specified favored nodes: [jenkins-hbase3.apache.org:44853] 2023-08-10T06:34:38,173 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:41489/backupUT 2023-08-10T06:34:38,174 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1691649269054 2023-08-10T06:34:38,174 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-08-10T06:34:38,179 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:41489/backupUT/.tmp/backup_1691649269054, hdfs://localhost:41489/backupUT] 2023-08-10T06:34:38,205 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-08-10T06:34:38,816 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-08-10T06:34:38,818 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1691649269054 2023-08-10T06:34:38,818 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 9114 bytes copied." 2023-08-10T06:34:38,818 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local968326843_0006 completed: true true 2023-08-10T06:34:38,821 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=4412158 FILE: Number of bytes written=7472244 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=157494 HDFS: Number of bytes written=2359717 HDFS: Number of read operations=599 HDFS: Number of large read operations=0 HDFS: Number of write operations=256 Map-Reduce Framework Map input records=6 Map output records=0 Input split bytes=342 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1542455296 File Input Format Counters Bytes Read=1452 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=9114 Bytes Expected=9114 Files Copied=2 DIR_COPY=4 2023-08-10T06:34:38,821 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:41489/backupUT/.tmp/backup_1691649269054 to hdfs://localhost:41489/backupUT finished. 2023-08-10T06:34:38,831 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(433): deleted 0 original bulkload rows 2023-08-10T06:34:38,831 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649269054 2023-08-10T06:34:38,834 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649257759 to construct the dependency. 2023-08-10T06:34:38,835 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649257759 2023-08-10T06:34:38,837 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649257759/.backup.manifest 2023-08-10T06:34:38,837 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649257759BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:34:38,837 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(340): Got 2 ancestors for the current backup. 2023-08-10T06:34:38,837 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649269054 2023-08-10T06:34:38,839 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649257759 to construct the dependency. 2023-08-10T06:34:38,839 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649257759 2023-08-10T06:34:38,841 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649257759/.backup.manifest 2023-08-10T06:34:38,841 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649257759BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:34:38,841 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(340): Got 2 ancestors for the current backup. 2023-08-10T06:34:39,247 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:41489/backupUT/backup_1691649269054/.backup.manifest 2023-08-10T06:34:39,247 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649269054 2023-08-10T06:34:39,249 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649257759 to construct the dependency. 2023-08-10T06:34:39,249 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649257759 2023-08-10T06:34:39,252 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649257759/.backup.manifest 2023-08-10T06:34:39,252 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649257759BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:34:39,252 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(340): Got 2 ancestors for the current backup. 2023-08-10T06:34:39,253 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649269054 2023-08-10T06:34:39,255 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649257759 to construct the dependency. 2023-08-10T06:34:39,255 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649257759 2023-08-10T06:34:39,257 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649257759/.backup.manifest 2023-08-10T06:34:39,257 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649257759BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:34:39,257 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(340): Got 2 ancestors for the current backup. 2023-08-10T06:34:39,662 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:41489/backupUT/backup_1691649269054/.backup.manifest 2023-08-10T06:34:39,662 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649269054 2023-08-10T06:34:39,664 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649257759 to construct the dependency. 2023-08-10T06:34:39,664 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649257759 2023-08-10T06:34:39,667 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649257759/.backup.manifest 2023-08-10T06:34:39,667 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649257759BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:34:39,667 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(340): Got 2 ancestors for the current backup. 2023-08-10T06:34:40,073 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:41489/backupUT/backup_1691649269054/.backup.manifest 2023-08-10T06:34:40,073 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(358): Backup backup_1691649269054 finished: type=INCREMENTAL,tablelist=ns2:test-16916492264741;test-1691649226474,targetRootDir=hdfs://localhost:41489/backupUT,startts=1691649273203,completets=1691649278831,bytescopied=0 2023-08-10T06:34:40,073 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:34:40,078 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:34:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:34:40,081 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:34:40,082 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:34:40,086 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(376): Backup backup_1691649269054 completed. 2023-08-10T06:34:40,195 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:34:40,198 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:34:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649280198 (current time:1691649280198). 2023-08-10T06:34:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:34:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:34:40,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x06c1df14 to 127.0.0.1:59679 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:34:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72514bbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:34:40,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:34:40,209 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:35544, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:34:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x06c1df14 to 127.0.0.1:59679 2023-08-10T06:34:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:34:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:34:40,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure2.ProcedureExecutor(1033): Stored pid=42, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:34:40,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 42 2023-08-10T06:34:40,212 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:34:40,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-08-10T06:34:40,213 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:34:40,215 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:34:40,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-08-10T06:34:40,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-08-10T06:34:40,621 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:34:40,621 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47}] 2023-08-10T06:34:40,623 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:34:40,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44853 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=43 2023-08-10T06:34:40,774 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. 2023-08-10T06:34:40,775 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(2819): Flushing 60ed3292def93d2ecd5862b25318ed47 2/2 column families, dataSize=1.37 KB heapSize=2.52 KB 2023-08-10T06:34:40,779 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/meta/b7540cf2901649cc88f83d61ac6d6a1f is 171, key is trslm:hdfs://localhost:41489/backupUT\x00ns2:test-16916492264741/meta:log-roll-map/1691649278825/Put/seqid=0 2023-08-10T06:34:40,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-08-10T06:34:41,184 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=531 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/meta/b7540cf2901649cc88f83d61ac6d6a1f 2023-08-10T06:34:41,190 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: b7540cf2901649cc88f83d61ac6d6a1f 2023-08-10T06:34:41,193 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/session/f9a52bd725e64f909af23b5393aacba4 is 360, key is session:backup_1691649269054/session:context/1691649280081/Put/seqid=0 2023-08-10T06:34:41,272 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 2a873d0a0a9423038307a20a4f0e1113 changed from -1.0 to 0.0, refreshing cache 2023-08-10T06:34:41,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-08-10T06:34:41,597 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=873 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/session/f9a52bd725e64f909af23b5393aacba4 2023-08-10T06:34:41,604 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: f9a52bd725e64f909af23b5393aacba4 2023-08-10T06:34:41,605 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/meta/b7540cf2901649cc88f83d61ac6d6a1f as hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/b7540cf2901649cc88f83d61ac6d6a1f 2023-08-10T06:34:41,611 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/b7540cf2901649cc88f83d61ac6d6a1f, entries=4, sequenceid=39, filesize=5.7 K 2023-08-10T06:34:41,612 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/session/f9a52bd725e64f909af23b5393aacba4 as hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/f9a52bd725e64f909af23b5393aacba4 2023-08-10T06:34:41,621 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/f9a52bd725e64f909af23b5393aacba4, entries=2, sequenceid=39, filesize=5.3 K 2023-08-10T06:34:41,622 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(3022): Finished flush of dataSize ~1.37 KB/1404, heapSize ~2.49 KB/2552, currentSize=0 B/0 for 60ed3292def93d2ecd5862b25318ed47 in 847ms, sequenceid=39, compaction requested=true 2023-08-10T06:34:41,622 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(2520): Flush status journal for 60ed3292def93d2ecd5862b25318ed47: 2023-08-10T06:34:41,622 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. for snapshot_backup_system completed. 2023-08-10T06:34:41,622 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:34:41,622 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:34:41,622 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/ee5a7bc3977042ada997c293856052bb, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/d4291c086d8c4791bd00119e10f93814, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/b7540cf2901649cc88f83d61ac6d6a1f] hfiles 2023-08-10T06:34:41,622 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/ee5a7bc3977042ada997c293856052bb for snapshot=snapshot_backup_system 2023-08-10T06:34:41,622 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/d4291c086d8c4791bd00119e10f93814 for snapshot=snapshot_backup_system 2023-08-10T06:34:41,622 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/b7540cf2901649cc88f83d61ac6d6a1f for snapshot=snapshot_backup_system 2023-08-10T06:34:41,622 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/6743ea07cad64eb59ec77e7feafa1733, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/0eca886709dc4bc28ce13c5c7fbf8954, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/a9dfa7ceabf344a68958248aee65d7ab, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/f9a52bd725e64f909af23b5393aacba4] hfiles 2023-08-10T06:34:41,622 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/6743ea07cad64eb59ec77e7feafa1733 for snapshot=snapshot_backup_system 2023-08-10T06:34:41,622 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/0eca886709dc4bc28ce13c5c7fbf8954 for snapshot=snapshot_backup_system 2023-08-10T06:34:41,622 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/a9dfa7ceabf344a68958248aee65d7ab for snapshot=snapshot_backup_system 2023-08-10T06:34:41,622 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/f9a52bd725e64f909af23b5393aacba4 for snapshot=snapshot_backup_system 2023-08-10T06:34:42,028 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. 2023-08-10T06:34:42,028 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=43 2023-08-10T06:34:42,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.HMaster(3994): Remote procedure done, pid=43 2023-08-10T06:34:42,029 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:34:42,030 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:34:42,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=43, resume processing ppid=42 2023-08-10T06:34:42,032 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:34:42,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=43, ppid=42, state=SUCCESS; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47 in 1.4100 sec 2023-08-10T06:34:42,033 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:34:42,033 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:34:42,033 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:34:42,035 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:34:42,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-08-10T06:34:42,446 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:34:42,456 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:34:42,457 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:34:43,278 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:34:43,278 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 42 2023-08-10T06:34:43,280 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=42, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0680 sec 2023-08-10T06:34:44,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-08-10T06:34:44,319 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 42 completed 2023-08-10T06:34:44,319 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(115): Backup backup_1691649280173 started at 1691649284319. 2023-08-10T06:34:44,321 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(119): Backup session backup_1691649280173 has been started. 2023-08-10T06:34:44,322 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalTableBackupClient(268): For incremental backup, current table set is [test-1691649226474, ns2:test-16916492264741] 2023-08-10T06:34:44,329 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(76): StartCode 1691649261963for backupID backup_1691649280173 2023-08-10T06:34:44,329 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-08-10T06:34:44,330 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-08-10T06:34:44,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-08-10T06:34:44,330 INFO [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-08-10T06:34:44,330 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-08-10T06:34:44,331 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-08-10T06:34:44,331 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-08-10T06:34:44,332 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-08-10T06:34:44,332 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:34:44,333 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-08-10T06:34:44,333 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:44,333 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:34:44,333 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:34:44,334 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:44,334 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-08-10T06:34:44,334 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-08-10T06:34:44,334 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-08-10T06:34:44,335 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-08-10T06:34:44,335 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:34:44,335 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-08-10T06:34:44,335 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-08-10T06:34:44,335 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-08-10T06:34:44,335 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-08-10T06:34:44,335 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-08-10T06:34:44,336 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-08-10T06:34:44,336 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-08-10T06:34:44,336 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,44853,1691649217899' joining acquired barrier for procedure (rolllog) in zk 2023-08-10T06:34:44,339 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:44,339 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:44,339 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:44,339 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:34:44,339 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:34:44,339 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-08-10T06:34:44,339 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-08-10T06:34:44,339 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:34:44,340 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:34:44,340 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:34:44,340 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:44,341 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:34:44,341 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,44853,1691649217899' joining acquired barrier for procedure 'rolllog' on coordinator 2023-08-10T06:34:44,341 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@773b8c31[Count = 0] remaining members to acquire global barrier 2023-08-10T06:34:44,341 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-08-10T06:34:44,341 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:44,342 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:44,342 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:44,342 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:44,342 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-08-10T06:34:44,343 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:44,343 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-08-10T06:34:44,343 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-08-10T06:34:44,343 DEBUG [rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:44,343 INFO [rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1691649273257 highest: 1691649273257 on jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:44,343 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta:.meta(num 1691649273234) roll requested 2023-08-10T06:34:44,354 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649284343.meta, exclude list is [], retry=0 2023-08-10T06:34:44,360 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK] 2023-08-10T06:34:44,361 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649273234.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649284343.meta 2023-08-10T06:34:44,361 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK]] 2023-08-10T06:34:44,361 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649273234.meta is not closed yet, will try archiving it next time 2023-08-10T06:34:44,361 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44853%2C1691649217899:(num 1691649273257) roll requested 2023-08-10T06:34:44,364 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649273234.meta to hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649273234.meta 2023-08-10T06:34:44,374 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649284361, exclude list is [], retry=0 2023-08-10T06:34:44,377 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK] 2023-08-10T06:34:44,379 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649273257 with entries=109, filesize=13.07 KB; new WAL /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649284361 2023-08-10T06:34:44,379 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK]] 2023-08-10T06:34:44,379 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649273257 is not closed yet, will try archiving it next time 2023-08-10T06:34:44,381 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649273257 not finished, retry = 0 2023-08-10T06:34:44,383 DEBUG [rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-08-10T06:34:44,383 INFO [rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1691649284361 on jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:44,391 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-08-10T06:34:44,391 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,44853,1691649217899' in zk 2023-08-10T06:34:44,395 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:44,395 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-08-10T06:34:44,395 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:44,395 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:34:44,395 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:34:44,395 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:34:44,395 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-08-10T06:34:44,396 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:34:44,397 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:34:44,397 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:34:44,397 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:44,397 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:34:44,398 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:34:44,398 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:44,399 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,44853,1691649217899': 2023-08-10T06:34:44,399 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,44853,1691649217899' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-08-10T06:34:44,399 INFO [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-08-10T06:34:44,399 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-08-10T06:34:44,399 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-08-10T06:34:44,399 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-08-10T06:34:44,399 INFO [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-08-10T06:34:44,401 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:34:44,401 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:34:44,401 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-08-10T06:34:44,401 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-08-10T06:34:44,401 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-08-10T06:34:44,401 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:34:44,401 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:34:44,401 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:34:44,401 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:34:44,401 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:34:44,401 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:34:44,401 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:44,401 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:34:44,402 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:34:44,402 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:34:44,402 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:34:44,403 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:44,403 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:44,403 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:34:44,403 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:34:44,403 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:44,407 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:44,407 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-08-10T06:34:44,407 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-08-10T06:34:44,407 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-08-10T06:34:44,408 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:44,408 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-08-10T06:34:44,408 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:44,408 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:34:44,408 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:44,408 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:34:44,408 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:34:44,408 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2023-08-10T06:34:44,408 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:34:44,408 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-08-10T06:34:44,408 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-08-10T06:34:44,409 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin(2751): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-08-10T06:34:44,409 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:34:44,409 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin(2759): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-08-10T06:34:44,409 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:34:44,509 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin(2765): Getting current status of procedure from master... 2023-08-10T06:34:44,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-08-10T06:34:44,512 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:44853=1691649261963} newestTimestamps: {jenkins-hbase3.apache.org:44853=1691649273257} 2023-08-10T06:34:44,514 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649220360.meta 2023-08-10T06:34:44,514 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649284343.meta 2023-08-10T06:34:44,514 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649273257 2023-08-10T06:34:44,514 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649284361 2023-08-10T06:34:44,514 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649242397.meta 2023-08-10T06:34:44,514 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649261946.meta 2023-08-10T06:34:44,514 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649273234.meta 2023-08-10T06:34:44,925 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(146): Attempting to copy table info for:ns2:test-16916492264741 target: hdfs://localhost:41489/backupUT/backup_1691649280173/ns2/test-16916492264741 descriptor: 'ns2:test-16916492264741', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:34:44,925 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-08-10T06:34:44,927 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(151): Starting to write region info for table ns2:test-16916492264741 2023-08-10T06:34:45,333 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(158): Finished writing region info for table ns2:test-16916492264741 2023-08-10T06:34:45,744 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(146): Attempting to copy table info for:test-1691649226474 target: hdfs://localhost:41489/backupUT/backup_1691649280173/default/test-1691649226474 descriptor: 'test-1691649226474', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:34:45,744 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-08-10T06:34:45,746 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(151): Starting to write region info for table test-1691649226474 2023-08-10T06:34:46,153 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(158): Finished writing region info for table test-1691649226474 2023-08-10T06:34:46,158 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:41489/backupUT/.tmp/backup_1691649280173 from hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649273257 2023-08-10T06:34:46,159 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5884639a to 127.0.0.1:59679 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:34:46,166 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13ef0c37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:34:46,167 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:34:46,170 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:42794, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:34:46,172 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-08-10T06:34:46,172 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1691649226474 2023-08-10T06:34:46,176 DEBUG [hconnection-0x587e432c-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:34:46,178 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:35550, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:34:46,180 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1691649226474: test-1691649226474; 2023-08-10T06:34:46,180 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16916492264741 2023-08-10T06:34:46,182 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16916492264741: ns2:test-16916492264741; 2023-08-10T06:34:46,182 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(694): Configuring 2 reduce partitions to match current region count for all tables 2023-08-10T06:34:46,182 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/hbase-staging/partitions_031b8d63-7e5a-4d03-8d96-64336e30d39c 2023-08-10T06:34:46,593 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:46,593 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:46,594 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:46,594 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:47,563 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/hadoop-2113338355786102309.jar 2023-08-10T06:34:47,564 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:47,564 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:47,564 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:47,565 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:47,565 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:47,565 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:47,566 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:47,566 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:47,567 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:47,567 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:34:47,568 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:34:47,568 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:34:47,569 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:34:47,569 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:34:47,570 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:34:47,570 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:34:47,570 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:34:47,571 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:34:47,571 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:34:47,572 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:34:47,572 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:34:47,572 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:47,573 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:47,573 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:47,573 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:47,574 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:47,574 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:47,575 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:34:47,575 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: default:test-1691649226474,ns2:test-16916492264741 2023-08-10T06:34:47,575 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] client.ConnectionImplementation(2114): Closing master protocol: MasterService 2023-08-10T06:34:47,575 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5884639a to 127.0.0.1:59679 2023-08-10T06:34:47,575 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:34:47,576 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:34:48,623 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/hadoop-4556643105313119090.jar 2023-08-10T06:34:48,629 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-08-10T06:34:48,644 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649273257 for WAL files 2023-08-10T06:34:48,646 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649273257 2023-08-10T06:34:48,718 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/mapreduce.cluster.local.dir/1691649288683/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:34:48,718 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/mapreduce.cluster.local.dir/1691649288683/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:34:48,773 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649273257 for hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649273257 (-9223372036854775808:9223372036854775807) length:13396 2023-08-10T06:34:48,779 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-08-10T06:34:48,780 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-08-10T06:34:48,808 DEBUG [pool-228-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x35684147 to 127.0.0.1:59679 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:34:48,812 DEBUG [pool-228-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e71199f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:34:48,814 DEBUG [hconnection-0x7c31db67-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:34:48,816 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:35552, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:34:48,817 DEBUG [pool-228-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x35684147 to 127.0.0.1:59679 2023-08-10T06:34:48,818 DEBUG [pool-228-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:34:48,819 DEBUG [pool-228-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-t1100] 2023-08-10T06:34:48,819 DEBUG [pool-228-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-08-10T06:34:48,826 DEBUG [pool-228-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41489/backupUT/.tmp/backup_1691649280173/_temporary/0/_temporary/attempt_local2005068623_0007_r_000001_0/default/test-1691649226474/f/42b146daed5047f0a6f86e5a6950f1ea is 42, key is row-t1100/f:q1/1691649280098/Put/seqid=0 2023-08-10T06:34:48,828 WARN [Thread-1156 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:44853] Specified favored nodes: [jenkins-hbase3.apache.org:44853] 2023-08-10T06:34:49,750 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:41489/backupUT 2023-08-10T06:34:49,750 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1691649280173 2023-08-10T06:34:49,750 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-08-10T06:34:49,756 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:41489/backupUT/.tmp/backup_1691649280173, hdfs://localhost:41489/backupUT] 2023-08-10T06:34:49,779 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-08-10T06:34:50,357 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-08-10T06:34:50,359 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1691649280173 2023-08-10T06:34:50,359 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 9086 bytes copied." 2023-08-10T06:34:50,359 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local738801271_0008 completed: true true 2023-08-10T06:34:50,362 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=5052350 FILE: Number of bytes written=9876803 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=207030 HDFS: Number of bytes written=2395876 HDFS: Number of read operations=740 HDFS: Number of large read operations=0 HDFS: Number of write operations=303 Map-Reduce Framework Map input records=6 Map output records=0 Input split bytes=341 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1542455296 File Input Format Counters Bytes Read=1478 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=9086 Bytes Expected=9086 Files Copied=2 DIR_COPY=4 2023-08-10T06:34:50,362 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:41489/backupUT/.tmp/backup_1691649280173 to hdfs://localhost:41489/backupUT finished. 2023-08-10T06:34:50,371 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(433): deleted 0 original bulkload rows 2023-08-10T06:34:50,371 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649280173 2023-08-10T06:34:50,375 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649269054 to construct the dependency. 2023-08-10T06:34:50,375 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649269054 2023-08-10T06:34:50,378 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649269054/.backup.manifest 2023-08-10T06:34:50,378 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649269054BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:34:50,378 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649257759 to construct the dependency. 2023-08-10T06:34:50,378 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649257759 2023-08-10T06:34:50,380 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649257759/.backup.manifest 2023-08-10T06:34:50,380 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649257759BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:34:50,380 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(340): Got 3 ancestors for the current backup. 2023-08-10T06:34:50,380 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649280173 2023-08-10T06:34:50,382 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649269054 to construct the dependency. 2023-08-10T06:34:50,382 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649269054 2023-08-10T06:34:50,384 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649269054/.backup.manifest 2023-08-10T06:34:50,384 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649269054BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:34:50,384 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649257759 to construct the dependency. 2023-08-10T06:34:50,384 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649257759 2023-08-10T06:34:50,386 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649257759/.backup.manifest 2023-08-10T06:34:50,386 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649257759BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:34:50,387 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(340): Got 3 ancestors for the current backup. 2023-08-10T06:34:50,792 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:41489/backupUT/backup_1691649280173/.backup.manifest 2023-08-10T06:34:50,792 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649280173 2023-08-10T06:34:50,795 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649269054 to construct the dependency. 2023-08-10T06:34:50,795 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649269054 2023-08-10T06:34:50,797 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649269054/.backup.manifest 2023-08-10T06:34:50,797 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649269054BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:34:50,797 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649257759 to construct the dependency. 2023-08-10T06:34:50,797 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649257759 2023-08-10T06:34:50,800 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649257759/.backup.manifest 2023-08-10T06:34:50,800 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649257759BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:34:50,800 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(340): Got 3 ancestors for the current backup. 2023-08-10T06:34:50,800 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649280173 2023-08-10T06:34:50,802 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649269054 to construct the dependency. 2023-08-10T06:34:50,802 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649269054 2023-08-10T06:34:50,804 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649269054/.backup.manifest 2023-08-10T06:34:50,804 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649269054BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:34:50,805 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649257759 to construct the dependency. 2023-08-10T06:34:50,805 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649257759 2023-08-10T06:34:50,806 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649257759/.backup.manifest 2023-08-10T06:34:50,806 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649257759BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:34:50,806 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(340): Got 3 ancestors for the current backup. 2023-08-10T06:34:51,212 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:41489/backupUT/backup_1691649280173/.backup.manifest 2023-08-10T06:34:51,212 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649280173 2023-08-10T06:34:51,214 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649269054 to construct the dependency. 2023-08-10T06:34:51,214 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649269054 2023-08-10T06:34:51,217 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649269054/.backup.manifest 2023-08-10T06:34:51,217 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649269054BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:34:51,217 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649257759 to construct the dependency. 2023-08-10T06:34:51,217 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649257759 2023-08-10T06:34:51,219 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649257759/.backup.manifest 2023-08-10T06:34:51,219 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649257759BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:34:51,219 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(340): Got 3 ancestors for the current backup. 2023-08-10T06:34:51,624 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:41489/backupUT/backup_1691649280173/.backup.manifest 2023-08-10T06:34:51,624 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(358): Backup backup_1691649280173 finished: type=INCREMENTAL,tablelist=ns2:test-16916492264741;test-1691649226474,targetRootDir=hdfs://localhost:41489/backupUT,startts=1691649284319,completets=1691649290371,bytescopied=0 2023-08-10T06:34:51,624 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:34:51,629 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:34:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:34:51,632 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:34:51,634 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:34:51,636 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(376): Backup backup_1691649280173 completed. 2023-08-10T06:34:51,749 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:34:51,753 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:34:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649291753 (current time:1691649291753). 2023-08-10T06:34:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:34:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:34:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4cc59113 to 127.0.0.1:59679 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:34:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4422fcc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:34:51,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:34:51,762 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:40272, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:34:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4cc59113 to 127.0.0.1:59679 2023-08-10T06:34:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:34:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:34:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure2.ProcedureExecutor(1033): Stored pid=44, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:34:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 44 2023-08-10T06:34:51,765 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:34:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=44 2023-08-10T06:34:51,766 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:34:51,768 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:34:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=44 2023-08-10T06:34:52,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=44 2023-08-10T06:34:52,174 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:34:52,175 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47}] 2023-08-10T06:34:52,176 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=45, ppid=44, state=RUNNABLE; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:34:52,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44853 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=45 2023-08-10T06:34:52,328 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. 2023-08-10T06:34:52,328 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HRegion(2819): Flushing 60ed3292def93d2ecd5862b25318ed47 2/2 column families, dataSize=1.37 KB heapSize=2.52 KB 2023-08-10T06:34:52,332 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/meta/20be7fbcf544464eaf03f133b5735605 is 171, key is trslm:hdfs://localhost:41489/backupUT\x00ns2:test-16916492264741/meta:log-roll-map/1691649290366/Put/seqid=0 2023-08-10T06:34:52,337 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=531 B at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/meta/20be7fbcf544464eaf03f133b5735605 2023-08-10T06:34:52,342 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 20be7fbcf544464eaf03f133b5735605 2023-08-10T06:34:52,345 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/session/4b4b67835f074c859b336180b4bd2fe3 is 360, key is session:backup_1691649280173/session:context/1691649291633/Put/seqid=0 2023-08-10T06:34:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=44 2023-08-10T06:34:52,749 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=873 B at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/session/4b4b67835f074c859b336180b4bd2fe3 2023-08-10T06:34:52,755 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 4b4b67835f074c859b336180b4bd2fe3 2023-08-10T06:34:52,756 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/meta/20be7fbcf544464eaf03f133b5735605 as hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/20be7fbcf544464eaf03f133b5735605 2023-08-10T06:34:52,761 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/20be7fbcf544464eaf03f133b5735605, entries=4, sequenceid=50, filesize=5.7 K 2023-08-10T06:34:52,762 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/session/4b4b67835f074c859b336180b4bd2fe3 as hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/4b4b67835f074c859b336180b4bd2fe3 2023-08-10T06:34:52,768 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/4b4b67835f074c859b336180b4bd2fe3, entries=2, sequenceid=50, filesize=5.3 K 2023-08-10T06:34:52,769 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HRegion(3022): Finished flush of dataSize ~1.37 KB/1404, heapSize ~2.49 KB/2552, currentSize=0 B/0 for 60ed3292def93d2ecd5862b25318ed47 in 440ms, sequenceid=50, compaction requested=true 2023-08-10T06:34:52,769 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HRegion(2520): Flush status journal for 60ed3292def93d2ecd5862b25318ed47: 2023-08-10T06:34:52,769 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. for snapshot_backup_system completed. 2023-08-10T06:34:52,769 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:34:52,769 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:34:52,769 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/ee5a7bc3977042ada997c293856052bb, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/d4291c086d8c4791bd00119e10f93814, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/b7540cf2901649cc88f83d61ac6d6a1f, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/20be7fbcf544464eaf03f133b5735605] hfiles 2023-08-10T06:34:52,769 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/ee5a7bc3977042ada997c293856052bb for snapshot=snapshot_backup_system 2023-08-10T06:34:52,769 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/d4291c086d8c4791bd00119e10f93814 for snapshot=snapshot_backup_system 2023-08-10T06:34:52,769 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/b7540cf2901649cc88f83d61ac6d6a1f for snapshot=snapshot_backup_system 2023-08-10T06:34:52,769 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/20be7fbcf544464eaf03f133b5735605 for snapshot=snapshot_backup_system 2023-08-10T06:34:52,769 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/6743ea07cad64eb59ec77e7feafa1733, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/0eca886709dc4bc28ce13c5c7fbf8954, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/a9dfa7ceabf344a68958248aee65d7ab, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/f9a52bd725e64f909af23b5393aacba4, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/4b4b67835f074c859b336180b4bd2fe3] hfiles 2023-08-10T06:34:52,769 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/6743ea07cad64eb59ec77e7feafa1733 for snapshot=snapshot_backup_system 2023-08-10T06:34:52,769 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/0eca886709dc4bc28ce13c5c7fbf8954 for snapshot=snapshot_backup_system 2023-08-10T06:34:52,769 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/a9dfa7ceabf344a68958248aee65d7ab for snapshot=snapshot_backup_system 2023-08-10T06:34:52,769 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/f9a52bd725e64f909af23b5393aacba4 for snapshot=snapshot_backup_system 2023-08-10T06:34:52,769 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/4b4b67835f074c859b336180b4bd2fe3 for snapshot=snapshot_backup_system 2023-08-10T06:34:52,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=44 2023-08-10T06:34:53,179 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. 2023-08-10T06:34:53,179 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=45 2023-08-10T06:34:53,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.HMaster(3994): Remote procedure done, pid=45 2023-08-10T06:34:53,179 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:34:53,180 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=45, ppid=44, state=RUNNABLE; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:34:53,183 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=45, resume processing ppid=44 2023-08-10T06:34:53,183 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:34:53,183 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=45, ppid=44, state=SUCCESS; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47 in 1.0060 sec 2023-08-10T06:34:53,184 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:34:53,185 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:34:53,185 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:34:53,185 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:34:53,606 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:34:53,616 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:34:53,616 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:34:53,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=44 2023-08-10T06:34:54,434 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:34:54,435 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 44 2023-08-10T06:34:54,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=44, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.6720 sec 2023-08-10T06:34:55,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=44 2023-08-10T06:34:55,873 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 44 completed 2023-08-10T06:34:55,873 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(115): Backup backup_1691649291725 started at 1691649295873. 2023-08-10T06:34:55,874 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(119): Backup session backup_1691649291725 has been started. 2023-08-10T06:34:55,876 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalTableBackupClient(268): For incremental backup, current table set is [test-1691649226474, ns2:test-16916492264741] 2023-08-10T06:34:55,882 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(76): StartCode 1691649273257for backupID backup_1691649291725 2023-08-10T06:34:55,882 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-08-10T06:34:55,883 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-08-10T06:34:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-08-10T06:34:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-08-10T06:34:55,883 INFO [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-08-10T06:34:55,884 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-08-10T06:34:55,885 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-08-10T06:34:55,885 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-08-10T06:34:55,886 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-08-10T06:34:55,886 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:34:55,887 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-08-10T06:34:55,887 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:55,887 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:34:55,887 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:34:55,887 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:55,887 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-08-10T06:34:55,888 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-08-10T06:34:55,888 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-08-10T06:34:55,888 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-08-10T06:34:55,888 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:34:55,888 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-08-10T06:34:55,889 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-08-10T06:34:55,889 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-08-10T06:34:55,889 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-08-10T06:34:55,889 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-08-10T06:34:55,889 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-08-10T06:34:55,890 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-08-10T06:34:55,890 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-08-10T06:34:55,890 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,44853,1691649217899' joining acquired barrier for procedure (rolllog) in zk 2023-08-10T06:34:55,892 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:55,892 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:55,892 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:55,892 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:34:55,892 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:34:55,893 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-08-10T06:34:55,893 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-08-10T06:34:55,893 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:34:55,893 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:34:55,893 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:34:55,894 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:55,894 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:34:55,894 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,44853,1691649217899' joining acquired barrier for procedure 'rolllog' on coordinator 2023-08-10T06:34:55,895 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@7378804e[Count = 0] remaining members to acquire global barrier 2023-08-10T06:34:55,895 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-08-10T06:34:55,895 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:55,896 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:55,896 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:55,896 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:55,896 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:55,896 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-08-10T06:34:55,897 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-08-10T06:34:55,897 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-08-10T06:34:55,897 DEBUG [rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:55,897 INFO [rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1691649284361 highest: 1691649284361 on jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:55,902 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta:.meta(num 1691649284343) roll requested 2023-08-10T06:34:55,913 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649295902.meta, exclude list is [], retry=0 2023-08-10T06:34:55,917 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK] 2023-08-10T06:34:55,919 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649284343.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649295902.meta 2023-08-10T06:34:55,919 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK]] 2023-08-10T06:34:55,919 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649284343.meta is not closed yet, will try archiving it next time 2023-08-10T06:34:55,919 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44853%2C1691649217899:(num 1691649284361) roll requested 2023-08-10T06:34:55,921 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649284343.meta not finished, retry = 0 2023-08-10T06:34:55,931 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649295919, exclude list is [], retry=0 2023-08-10T06:34:55,934 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK] 2023-08-10T06:34:55,936 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649284361 with entries=109, filesize=13.56 KB; new WAL /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649295919 2023-08-10T06:34:55,936 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK]] 2023-08-10T06:34:55,936 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649284361 is not closed yet, will try archiving it next time 2023-08-10T06:34:55,938 DEBUG [rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 41 2023-08-10T06:34:55,938 INFO [rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1691649295919 on jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:55,947 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-08-10T06:34:55,947 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,44853,1691649217899' in zk 2023-08-10T06:34:55,949 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:55,949 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-08-10T06:34:55,949 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:34:55,949 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-08-10T06:34:55,950 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:55,950 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:34:55,950 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:34:55,950 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:34:55,951 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:34:55,951 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:34:55,951 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:55,951 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:34:55,952 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:34:55,952 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:55,952 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,44853,1691649217899': 2023-08-10T06:34:55,953 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,44853,1691649217899' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-08-10T06:34:55,953 INFO [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-08-10T06:34:55,953 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-08-10T06:34:55,953 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-08-10T06:34:55,953 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-08-10T06:34:55,953 INFO [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-08-10T06:34:55,954 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:34:55,954 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:34:55,954 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-08-10T06:34:55,954 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-08-10T06:34:55,955 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-08-10T06:34:55,955 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:34:55,955 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:34:55,955 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:34:55,955 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:34:55,955 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:34:55,955 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:34:55,955 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:55,955 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:34:55,955 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:34:55,956 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:34:55,956 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:34:55,957 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:55,957 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:55,957 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:34:55,958 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:34:55,958 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:55,966 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-08-10T06:34:55,966 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-08-10T06:34:55,966 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:55,966 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:34:55,966 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:34:55,966 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-08-10T06:34:55,966 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:34:55,966 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-08-10T06:34:55,966 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:34:55,967 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:55,967 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:34:55,967 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:34:55,966 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-08-10T06:34:55,967 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-08-10T06:34:55,967 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:34:55,967 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:34:55,967 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin(2751): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-08-10T06:34:55,967 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin(2759): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-08-10T06:34:56,022 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649284343.meta to hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649284343.meta 2023-08-10T06:34:56,067 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin(2765): Getting current status of procedure from master... 2023-08-10T06:34:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-08-10T06:34:56,071 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:44853=1691649273257} newestTimestamps: {jenkins-hbase3.apache.org:44853=1691649284361} 2023-08-10T06:34:56,072 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649220360.meta 2023-08-10T06:34:56,072 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649295902.meta 2023-08-10T06:34:56,072 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649284361 2023-08-10T06:34:56,072 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649295919 2023-08-10T06:34:56,073 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649242397.meta 2023-08-10T06:34:56,073 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649261946.meta 2023-08-10T06:34:56,073 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649273234.meta 2023-08-10T06:34:56,073 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649284343.meta 2023-08-10T06:34:56,482 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(146): Attempting to copy table info for:ns2:test-16916492264741 target: hdfs://localhost:41489/backupUT/backup_1691649291725/ns2/test-16916492264741 descriptor: 'ns2:test-16916492264741', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:34:56,482 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-08-10T06:34:56,485 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(151): Starting to write region info for table ns2:test-16916492264741 2023-08-10T06:34:56,890 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(158): Finished writing region info for table ns2:test-16916492264741 2023-08-10T06:34:57,301 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(146): Attempting to copy table info for:test-1691649226474 target: hdfs://localhost:41489/backupUT/backup_1691649291725/default/test-1691649226474 descriptor: 'test-1691649226474', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:34:57,301 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-08-10T06:34:57,303 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(151): Starting to write region info for table test-1691649226474 2023-08-10T06:34:57,709 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(158): Finished writing region info for table test-1691649226474 2023-08-10T06:34:57,714 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:41489/backupUT/.tmp/backup_1691649291725 from hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649284361 2023-08-10T06:34:57,714 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3880e9e9 to 127.0.0.1:59679 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:34:57,723 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67091011, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:34:57,725 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:34:57,726 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:45440, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:34:57,727 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-08-10T06:34:57,727 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1691649226474 2023-08-10T06:34:57,730 DEBUG [hconnection-0x78de6025-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:34:57,732 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:40284, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:34:57,733 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1691649226474: test-1691649226474; 2023-08-10T06:34:57,733 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16916492264741 2023-08-10T06:34:57,735 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16916492264741: ns2:test-16916492264741; 2023-08-10T06:34:57,735 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(694): Configuring 2 reduce partitions to match current region count for all tables 2023-08-10T06:34:57,735 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/hbase-staging/partitions_92de0f58-0137-408c-9171-3900191f8815 2023-08-10T06:34:58,142 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:58,143 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:58,144 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:58,144 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:59,121 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/hadoop-363731133872256517.jar 2023-08-10T06:34:59,121 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:59,121 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:59,122 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:59,122 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:59,123 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:59,123 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:59,123 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:59,124 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:59,124 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:59,125 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:34:59,125 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:34:59,126 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:34:59,126 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:34:59,127 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:34:59,127 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:34:59,127 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:34:59,128 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:34:59,128 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:34:59,128 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:34:59,129 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:34:59,129 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:34:59,130 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:59,130 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:59,130 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:59,131 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:59,131 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:59,131 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:34:59,132 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:34:59,132 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: default:test-1691649226474,ns2:test-16916492264741 2023-08-10T06:34:59,132 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] client.ConnectionImplementation(2114): Closing master protocol: MasterService 2023-08-10T06:34:59,132 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3880e9e9 to 127.0.0.1:59679 2023-08-10T06:34:59,132 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:34:59,133 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:35:00,102 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/hadoop-5728332754479939332.jar 2023-08-10T06:35:00,107 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-08-10T06:35:00,122 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649284361 for WAL files 2023-08-10T06:35:00,123 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649284361 2023-08-10T06:35:00,196 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/mapreduce.cluster.local.dir/1691649300158/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:35:00,197 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/mapreduce.cluster.local.dir/1691649300158/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:35:00,275 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649284361 for hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649284361 (-9223372036854775808:9223372036854775807) length:13891 2023-08-10T06:35:00,280 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-08-10T06:35:00,280 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-08-10T06:35:00,295 DEBUG [pool-241-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x42a14636 to 127.0.0.1:59679 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:35:00,300 DEBUG [pool-241-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61dbbae2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:35:00,302 DEBUG [hconnection-0x5fb4cd47-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:35:00,303 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:45236, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:35:00,304 DEBUG [pool-241-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x42a14636 to 127.0.0.1:59679 2023-08-10T06:35:00,304 DEBUG [pool-241-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:35:00,305 DEBUG [pool-241-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-t1100] 2023-08-10T06:35:00,305 DEBUG [pool-241-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-08-10T06:35:00,312 DEBUG [pool-241-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41489/backupUT/.tmp/backup_1691649291725/_temporary/0/_temporary/attempt_local1399845402_0009_r_000000_0/ns2/test-16916492264741/f/33505d60a3d44790ba2b4c3478b6e391 is 42, key is row-t1100/f:q1/1691649291648/Put/seqid=0 2023-08-10T06:35:00,314 WARN [Thread-1289 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:44853] Specified favored nodes: [jenkins-hbase3.apache.org:44853] 2023-08-10T06:35:01,230 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:41489/backupUT 2023-08-10T06:35:01,231 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1691649291725 2023-08-10T06:35:01,231 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-08-10T06:35:01,236 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:41489/backupUT/.tmp/backup_1691649291725, hdfs://localhost:41489/backupUT] 2023-08-10T06:35:01,260 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-08-10T06:35:01,839 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-08-10T06:35:01,841 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1691649291725 2023-08-10T06:35:01,841 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 9405 bytes copied." 2023-08-10T06:35:01,842 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local763999942_0010 completed: true true 2023-08-10T06:35:01,845 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=5689291 FILE: Number of bytes written=12282263 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=261732 HDFS: Number of bytes written=2437040 HDFS: Number of read operations=892 HDFS: Number of large read operations=0 HDFS: Number of write operations=351 Map-Reduce Framework Map input records=6 Map output records=0 Input split bytes=340 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1555562496 File Input Format Counters Bytes Read=1452 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=9405 Bytes Expected=9405 Files Copied=2 DIR_COPY=4 2023-08-10T06:35:01,845 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:41489/backupUT/.tmp/backup_1691649291725 to hdfs://localhost:41489/backupUT finished. 2023-08-10T06:35:01,855 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(433): deleted 0 original bulkload rows 2023-08-10T06:35:01,855 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649291725 2023-08-10T06:35:01,859 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649280173 to construct the dependency. 2023-08-10T06:35:01,859 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649280173 2023-08-10T06:35:01,862 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649280173/.backup.manifest 2023-08-10T06:35:01,862 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649280173BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:01,862 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649269054 to construct the dependency. 2023-08-10T06:35:01,862 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649269054 2023-08-10T06:35:01,864 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649269054/.backup.manifest 2023-08-10T06:35:01,864 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649269054BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:01,864 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649257759 to construct the dependency. 2023-08-10T06:35:01,864 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649257759 2023-08-10T06:35:01,866 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649257759/.backup.manifest 2023-08-10T06:35:01,866 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649257759BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:01,866 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(340): Got 4 ancestors for the current backup. 2023-08-10T06:35:01,866 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649291725 2023-08-10T06:35:01,868 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649280173 to construct the dependency. 2023-08-10T06:35:01,868 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649280173 2023-08-10T06:35:01,870 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649280173/.backup.manifest 2023-08-10T06:35:01,870 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649280173BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:01,871 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649269054 to construct the dependency. 2023-08-10T06:35:01,871 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649269054 2023-08-10T06:35:01,873 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649269054/.backup.manifest 2023-08-10T06:35:01,873 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649269054BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:01,873 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649257759 to construct the dependency. 2023-08-10T06:35:01,873 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649257759 2023-08-10T06:35:01,875 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649257759/.backup.manifest 2023-08-10T06:35:01,875 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649257759BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:01,875 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(340): Got 4 ancestors for the current backup. 2023-08-10T06:35:02,283 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:41489/backupUT/backup_1691649291725/.backup.manifest 2023-08-10T06:35:02,283 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649291725 2023-08-10T06:35:02,286 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649280173 to construct the dependency. 2023-08-10T06:35:02,286 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649280173 2023-08-10T06:35:02,289 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649280173/.backup.manifest 2023-08-10T06:35:02,289 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649280173BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:02,289 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649269054 to construct the dependency. 2023-08-10T06:35:02,289 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649269054 2023-08-10T06:35:02,291 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649269054/.backup.manifest 2023-08-10T06:35:02,291 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649269054BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:02,291 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649257759 to construct the dependency. 2023-08-10T06:35:02,291 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649257759 2023-08-10T06:35:02,293 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649257759/.backup.manifest 2023-08-10T06:35:02,293 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649257759BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:02,293 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(340): Got 4 ancestors for the current backup. 2023-08-10T06:35:02,293 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649291725 2023-08-10T06:35:02,295 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649280173 to construct the dependency. 2023-08-10T06:35:02,295 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649280173 2023-08-10T06:35:02,297 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649280173/.backup.manifest 2023-08-10T06:35:02,297 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649280173BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:02,297 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649269054 to construct the dependency. 2023-08-10T06:35:02,298 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649269054 2023-08-10T06:35:02,299 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649269054/.backup.manifest 2023-08-10T06:35:02,299 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649269054BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:02,299 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649257759 to construct the dependency. 2023-08-10T06:35:02,299 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649257759 2023-08-10T06:35:02,301 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649257759/.backup.manifest 2023-08-10T06:35:02,301 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649257759BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:02,301 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(340): Got 4 ancestors for the current backup. 2023-08-10T06:35:02,706 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:41489/backupUT/backup_1691649291725/.backup.manifest 2023-08-10T06:35:02,707 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649291725 2023-08-10T06:35:02,709 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649280173 to construct the dependency. 2023-08-10T06:35:02,709 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649280173 2023-08-10T06:35:02,712 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649280173/.backup.manifest 2023-08-10T06:35:02,712 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649280173BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:02,712 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649269054 to construct the dependency. 2023-08-10T06:35:02,712 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649269054 2023-08-10T06:35:02,714 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649269054/.backup.manifest 2023-08-10T06:35:02,714 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649269054BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:02,714 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649257759 to construct the dependency. 2023-08-10T06:35:02,714 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649257759 2023-08-10T06:35:02,716 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649257759/.backup.manifest 2023-08-10T06:35:02,716 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649257759BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:02,717 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(340): Got 4 ancestors for the current backup. 2023-08-10T06:35:03,123 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:41489/backupUT/backup_1691649291725/.backup.manifest 2023-08-10T06:35:03,123 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(358): Backup backup_1691649291725 finished: type=INCREMENTAL,tablelist=ns2:test-16916492264741;test-1691649226474,targetRootDir=hdfs://localhost:41489/backupUT,startts=1691649295873,completets=1691649301855,bytescopied=0 2023-08-10T06:35:03,123 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:35:03,128 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:35:03,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:35:03,131 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:35:03,132 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:35:03,135 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(376): Backup backup_1691649291725 completed. 2023-08-10T06:35:03,155 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:35:03,158 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:35:03,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649303158 (current time:1691649303158). 2023-08-10T06:35:03,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:35:03,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:35:03,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x579e70bc to 127.0.0.1:59679 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:35:03,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0692a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:35:03,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:35:03,167 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:45252, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:35:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x579e70bc to 127.0.0.1:59679 2023-08-10T06:35:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:35:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:35:03,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure2.ProcedureExecutor(1033): Stored pid=46, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:35:03,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 46 2023-08-10T06:35:03,170 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:35:03,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-08-10T06:35:03,171 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:35:03,173 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:35:03,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-08-10T06:35:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-08-10T06:35:03,579 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:35:03,579 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47}] 2023-08-10T06:35:03,581 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=47, ppid=46, state=RUNNABLE; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:35:03,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44853 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=47 2023-08-10T06:35:03,733 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. 2023-08-10T06:35:03,733 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.HRegion(2819): Flushing 60ed3292def93d2ecd5862b25318ed47 2/2 column families, dataSize=1.37 KB heapSize=2.52 KB 2023-08-10T06:35:03,737 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/meta/10815475db68465796bf1f8cdb79c442 is 171, key is trslm:hdfs://localhost:41489/backupUT\x00ns2:test-16916492264741/meta:log-roll-map/1691649301849/Put/seqid=0 2023-08-10T06:35:03,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-08-10T06:35:04,141 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=531 B at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/meta/10815475db68465796bf1f8cdb79c442 2023-08-10T06:35:04,147 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 10815475db68465796bf1f8cdb79c442 2023-08-10T06:35:04,149 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/session/a642f64893574ae7b70f03df104c31d7 is 360, key is session:backup_1691649291725/session:context/1691649303131/Put/seqid=0 2023-08-10T06:35:04,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-08-10T06:35:04,554 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=873 B at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/session/a642f64893574ae7b70f03df104c31d7 2023-08-10T06:35:04,559 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a642f64893574ae7b70f03df104c31d7 2023-08-10T06:35:04,560 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/meta/10815475db68465796bf1f8cdb79c442 as hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/10815475db68465796bf1f8cdb79c442 2023-08-10T06:35:04,564 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/10815475db68465796bf1f8cdb79c442, entries=4, sequenceid=61, filesize=5.7 K 2023-08-10T06:35:04,565 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/session/a642f64893574ae7b70f03df104c31d7 as hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/a642f64893574ae7b70f03df104c31d7 2023-08-10T06:35:04,571 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/a642f64893574ae7b70f03df104c31d7, entries=2, sequenceid=61, filesize=5.3 K 2023-08-10T06:35:04,572 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.HRegion(3022): Finished flush of dataSize ~1.37 KB/1404, heapSize ~2.49 KB/2552, currentSize=0 B/0 for 60ed3292def93d2ecd5862b25318ed47 in 838ms, sequenceid=61, compaction requested=true 2023-08-10T06:35:04,572 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.HRegion(2520): Flush status journal for 60ed3292def93d2ecd5862b25318ed47: 2023-08-10T06:35:04,572 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. for snapshot_backup_system completed. 2023-08-10T06:35:04,572 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:35:04,572 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:35:04,572 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/ee5a7bc3977042ada997c293856052bb, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/d4291c086d8c4791bd00119e10f93814, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/b7540cf2901649cc88f83d61ac6d6a1f, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/20be7fbcf544464eaf03f133b5735605, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/10815475db68465796bf1f8cdb79c442] hfiles 2023-08-10T06:35:04,572 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/ee5a7bc3977042ada997c293856052bb for snapshot=snapshot_backup_system 2023-08-10T06:35:04,572 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/d4291c086d8c4791bd00119e10f93814 for snapshot=snapshot_backup_system 2023-08-10T06:35:04,572 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/b7540cf2901649cc88f83d61ac6d6a1f for snapshot=snapshot_backup_system 2023-08-10T06:35:04,572 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/20be7fbcf544464eaf03f133b5735605 for snapshot=snapshot_backup_system 2023-08-10T06:35:04,572 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/10815475db68465796bf1f8cdb79c442 for snapshot=snapshot_backup_system 2023-08-10T06:35:04,572 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/6743ea07cad64eb59ec77e7feafa1733, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/0eca886709dc4bc28ce13c5c7fbf8954, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/a9dfa7ceabf344a68958248aee65d7ab, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/f9a52bd725e64f909af23b5393aacba4, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/4b4b67835f074c859b336180b4bd2fe3, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/a642f64893574ae7b70f03df104c31d7] hfiles 2023-08-10T06:35:04,572 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (1/6): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/6743ea07cad64eb59ec77e7feafa1733 for snapshot=snapshot_backup_system 2023-08-10T06:35:04,572 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (2/6): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/0eca886709dc4bc28ce13c5c7fbf8954 for snapshot=snapshot_backup_system 2023-08-10T06:35:04,572 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (3/6): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/a9dfa7ceabf344a68958248aee65d7ab for snapshot=snapshot_backup_system 2023-08-10T06:35:04,572 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (4/6): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/f9a52bd725e64f909af23b5393aacba4 for snapshot=snapshot_backup_system 2023-08-10T06:35:04,572 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (5/6): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/4b4b67835f074c859b336180b4bd2fe3 for snapshot=snapshot_backup_system 2023-08-10T06:35:04,572 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (6/6): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/a642f64893574ae7b70f03df104c31d7 for snapshot=snapshot_backup_system 2023-08-10T06:35:04,977 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. 2023-08-10T06:35:04,978 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=47 2023-08-10T06:35:04,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.HMaster(3994): Remote procedure done, pid=47 2023-08-10T06:35:04,978 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:35:04,979 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=47, ppid=46, state=RUNNABLE; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:35:04,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=47, resume processing ppid=46 2023-08-10T06:35:04,982 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:35:04,982 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=47, ppid=46, state=SUCCESS; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47 in 1.4010 sec 2023-08-10T06:35:04,982 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:35:04,983 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:35:04,983 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:35:04,983 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:35:05,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-08-10T06:35:05,393 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:35:05,403 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:35:05,403 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:35:06,221 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:35:06,221 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 46 2023-08-10T06:35:06,222 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=46, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0530 sec 2023-08-10T06:35:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-08-10T06:35:07,278 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 46 completed 2023-08-10T06:35:07,278 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(115): Backup backup_1691649303144 started at 1691649307278. 2023-08-10T06:35:07,279 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(119): Backup session backup_1691649303144 has been started. 2023-08-10T06:35:07,284 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-08-10T06:35:07,284 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-08-10T06:35:07,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-08-10T06:35:07,285 INFO [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-08-10T06:35:07,285 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-08-10T06:35:07,286 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-08-10T06:35:07,286 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-08-10T06:35:07,286 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-08-10T06:35:07,286 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:35:07,290 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:07,290 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-08-10T06:35:07,290 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:35:07,290 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:35:07,290 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:07,290 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-08-10T06:35:07,290 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-08-10T06:35:07,291 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-08-10T06:35:07,291 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-08-10T06:35:07,291 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:35:07,291 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-08-10T06:35:07,291 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-08-10T06:35:07,291 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-08-10T06:35:07,291 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-08-10T06:35:07,291 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-08-10T06:35:07,292 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-08-10T06:35:07,292 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-08-10T06:35:07,292 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,44853,1691649217899' joining acquired barrier for procedure (rolllog) in zk 2023-08-10T06:35:07,294 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:07,294 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-08-10T06:35:07,294 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:07,294 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:35:07,294 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:35:07,294 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-08-10T06:35:07,295 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-08-10T06:35:07,295 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:35:07,295 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:35:07,295 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:35:07,295 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:07,296 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:35:07,296 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,44853,1691649217899' joining acquired barrier for procedure 'rolllog' on coordinator 2023-08-10T06:35:07,296 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@112c3b8[Count = 0] remaining members to acquire global barrier 2023-08-10T06:35:07,296 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-08-10T06:35:07,296 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-08-10T06:35:07,297 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:35:07,297 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-08-10T06:35:07,297 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-08-10T06:35:07,297 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-08-10T06:35:07,297 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:07,297 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-08-10T06:35:07,297 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-08-10T06:35:07,298 DEBUG [rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:07,298 INFO [rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1691649295919 highest: 1691649295919 on jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:07,298 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta:.meta(num 1691649295902) roll requested 2023-08-10T06:35:07,309 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649307298.meta, exclude list is [], retry=0 2023-08-10T06:35:07,314 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK] 2023-08-10T06:35:07,316 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649295902.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649307298.meta 2023-08-10T06:35:07,316 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK]] 2023-08-10T06:35:07,316 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649295902.meta is not closed yet, will try archiving it next time 2023-08-10T06:35:07,316 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44853%2C1691649217899:(num 1691649295919) roll requested 2023-08-10T06:35:07,318 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649295902.meta not finished, retry = 0 2023-08-10T06:35:07,327 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649307316, exclude list is [], retry=0 2023-08-10T06:35:07,330 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK] 2023-08-10T06:35:07,331 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649295919 with entries=10, filesize=2.51 KB; new WAL /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649307316 2023-08-10T06:35:07,331 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK]] 2023-08-10T06:35:07,331 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649295919 is not closed yet, will try archiving it next time 2023-08-10T06:35:07,338 DEBUG [rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-08-10T06:35:07,338 INFO [rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1691649307316 on jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:07,345 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-08-10T06:35:07,345 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,44853,1691649217899' in zk 2023-08-10T06:35:07,349 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-08-10T06:35:07,349 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:07,349 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:35:07,349 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-08-10T06:35:07,349 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:07,349 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:35:07,349 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:35:07,350 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:35:07,350 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:35:07,350 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:35:07,351 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:07,351 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:35:07,351 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:35:07,351 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:07,352 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,44853,1691649217899': 2023-08-10T06:35:07,352 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,44853,1691649217899' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-08-10T06:35:07,352 INFO [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-08-10T06:35:07,352 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-08-10T06:35:07,352 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-08-10T06:35:07,352 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-08-10T06:35:07,352 INFO [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-08-10T06:35:07,354 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:35:07,354 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:35:07,354 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-08-10T06:35:07,354 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-08-10T06:35:07,354 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-08-10T06:35:07,354 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:35:07,354 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:35:07,354 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:35:07,354 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:35:07,354 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:35:07,354 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:35:07,354 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:07,355 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:35:07,355 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:35:07,355 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:35:07,355 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:35:07,356 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:07,356 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:07,356 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:35:07,356 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:35:07,357 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:07,360 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:07,360 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-08-10T06:35:07,360 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-08-10T06:35:07,360 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-08-10T06:35:07,360 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-08-10T06:35:07,360 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:07,360 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:35:07,360 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:35:07,360 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:35:07,360 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:35:07,360 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:35:07,360 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-08-10T06:35:07,360 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:35:07,360 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2023-08-10T06:35:07,361 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-08-10T06:35:07,361 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:35:07,361 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:35:07,361 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin(2751): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-08-10T06:35:07,361 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin(2759): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-08-10T06:35:07,418 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649295902.meta to hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649295902.meta 2023-08-10T06:35:07,461 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin(2765): Getting current status of procedure from master... 2023-08-10T06:35:07,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-08-10T06:35:07,465 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1691649307464_ns3_test-16916492264742 table=ns3:test-16916492264742 type=FLUSH ttl=0 } 2023-08-10T06:35:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649307465 (current time:1691649307465). 2023-08-10T06:35:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:35:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1691649307464_ns3_test-16916492264742 VERSION not specified, setting to 2 2023-08-10T06:35:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1ab42e94 to 127.0.0.1:59679 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:35:07,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e0bb6ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:35:07,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:35:07,475 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:45260, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:35:07,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1ab42e94 to 127.0.0.1:59679 2023-08-10T06:35:07,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:35:07,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:35:07,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure2.ProcedureExecutor(1033): Stored pid=48, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1691649307464_ns3_test-16916492264742 table=ns3:test-16916492264742 type=FLUSH ttl=0 } 2023-08-10T06:35:07,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1691649307464_ns3_test-16916492264742 table=ns3:test-16916492264742 type=FLUSH ttl=0 }, snapshot procedure id = 48 2023-08-10T06:35:07,479 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1691649307464_ns3_test-16916492264742 table=ns3:test-16916492264742 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:35:07,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=48 2023-08-10T06:35:07,480 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1691649307464_ns3_test-16916492264742 table=ns3:test-16916492264742 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:35:07,481 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1691649307464_ns3_test-16916492264742 table=ns3:test-16916492264742 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:35:07,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=48 2023-08-10T06:35:07,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=48 2023-08-10T06:35:07,888 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1691649307464_ns3_test-16916492264742 table=ns3:test-16916492264742 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:35:07,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE; SnapshotRegionProcedure a9e9ef626a68c685245826f3ca7f958f}] 2023-08-10T06:35:07,890 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=49, ppid=48, state=RUNNABLE; SnapshotRegionProcedure a9e9ef626a68c685245826f3ca7f958f 2023-08-10T06:35:08,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44853 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=49 2023-08-10T06:35:08,042 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16916492264742,,1691649230428.a9e9ef626a68c685245826f3ca7f958f. 2023-08-10T06:35:08,043 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegion(2520): Flush status journal for a9e9ef626a68c685245826f3ca7f958f: 2023-08-10T06:35:08,043 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16916492264742,,1691649230428.a9e9ef626a68c685245826f3ca7f958f. for snapshot_1691649307464_ns3_test-16916492264742 completed. 2023-08-10T06:35:08,043 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16916492264742,,1691649230428.a9e9ef626a68c685245826f3ca7f958f.' region-info for snapshot=snapshot_1691649307464_ns3_test-16916492264742 2023-08-10T06:35:08,043 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:35:08,043 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-08-10T06:35:08,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=48 2023-08-10T06:35:08,449 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16916492264742,,1691649230428.a9e9ef626a68c685245826f3ca7f958f. 2023-08-10T06:35:08,449 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=49 2023-08-10T06:35:08,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.HMaster(3994): Remote procedure done, pid=49 2023-08-10T06:35:08,450 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1691649307464_ns3_test-16916492264742 on region a9e9ef626a68c685245826f3ca7f958f 2023-08-10T06:35:08,451 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=49, ppid=48, state=RUNNABLE; SnapshotRegionProcedure a9e9ef626a68c685245826f3ca7f958f 2023-08-10T06:35:08,454 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=49, resume processing ppid=48 2023-08-10T06:35:08,454 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1691649307464_ns3_test-16916492264742 table=ns3:test-16916492264742 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:35:08,454 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=49, ppid=48, state=SUCCESS; SnapshotRegionProcedure a9e9ef626a68c685245826f3ca7f958f in 564 msec 2023-08-10T06:35:08,454 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1691649307464_ns3_test-16916492264742 table=ns3:test-16916492264742 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:35:08,455 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1691649307464_ns3_test-16916492264742 table=ns3:test-16916492264742 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:35:08,455 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1691649307464_ns3_test-16916492264742 2023-08-10T06:35:08,456 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/.tmp/snapshot_1691649307464_ns3_test-16916492264742 2023-08-10T06:35:08,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=48 2023-08-10T06:35:08,872 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1691649307464_ns3_test-16916492264742 table=ns3:test-16916492264742 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:35:08,877 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1691649307464_ns3_test-16916492264742 table=ns3:test-16916492264742 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:35:08,878 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/.tmp/snapshot_1691649307464_ns3_test-16916492264742 to hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/snapshot_1691649307464_ns3_test-16916492264742 2023-08-10T06:35:09,296 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1691649307464_ns3_test-16916492264742 table=ns3:test-16916492264742 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:35:09,296 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1691649307464_ns3_test-16916492264742 table=ns3:test-16916492264742 type=FLUSH ttl=0 }, snapshot procedure id = 48 2023-08-10T06:35:09,297 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=48, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1691649307464_ns3_test-16916492264742 table=ns3:test-16916492264742 type=FLUSH ttl=0 } in 1.8190 sec 2023-08-10T06:35:09,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=48 2023-08-10T06:35:09,585 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: ns3:test-16916492264742, procId: 48 completed 2023-08-10T06:35:09,585 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1691649303144 2023-08-10T06:35:09,585 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-08-10T06:35:09,585 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-08-10T06:35:09,586 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1691649303144_ns3:test-16916492264742 2023-08-10T06:35:09,586 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1691649307464_ns3_test-16916492264742 to hdfs://localhost:41489/backupUT/backup_1691649303144/ns3/test-16916492264742/ 2023-08-10T06:35:09,586 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-08-10T06:35:09,610 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:41489, inputRoot=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f 2023-08-10T06:35:09,610 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,37947,1691649214739_1155083354_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:41489/backupUT/backup_1691649303144/ns3/test-16916492264742, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:41489/backupUT/backup_1691649303144/ns3/test-16916492264742/.hbase-snapshot/.tmp/snapshot_1691649307464_ns3_test-16916492264742 2023-08-10T06:35:09,610 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:41489, inputRoot=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f, snapshotDir=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/snapshot_1691649307464_ns3_test-16916492264742. 2023-08-10T06:35:09,618 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/snapshot_1691649307464_ns3_test-16916492264742 to hdfs://localhost:41489/backupUT/backup_1691649303144/ns3/test-16916492264742/.hbase-snapshot/.tmp/snapshot_1691649307464_ns3_test-16916492264742 2023-08-10T06:35:10,033 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:10,033 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:10,034 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:10,034 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:11,017 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/hadoop-3343805602266374708.jar 2023-08-10T06:35:11,017 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:11,018 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:11,018 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:11,019 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:11,019 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:11,019 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:11,020 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:11,020 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:11,020 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:11,021 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:35:11,022 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:35:11,022 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:35:11,022 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:35:11,023 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:35:11,023 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:35:11,024 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:35:11,024 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:35:11,024 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:35:11,025 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:35:11,025 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:35:11,025 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:35:11,026 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:35:11,027 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:35:11,027 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:35:11,027 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:35:11,028 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:35:11,028 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:35:11,029 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:35:11,055 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-08-10T06:35:11,073 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1691649307464_ns3_test-16916492264742' hfile list 2023-08-10T06:35:11,150 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/mapreduce.cluster.local.dir/1691649311113/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:35:11,150 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/mapreduce.cluster.local.dir/1691649311113/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:35:12,180 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-08-10T06:35:12,181 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-08-10T06:35:12,186 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1691649307464_ns3_test-16916492264742 2023-08-10T06:35:12,186 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1691649307464_ns3_test-16916492264742 finished. 2023-08-10T06:35:12,186 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(1014): ns3:test-16916492264742 2023-08-10T06:35:12,192 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649303144 2023-08-10T06:35:12,192 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-08-10T06:35:12,597 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:41489/backupUT/backup_1691649303144/.backup.manifest 2023-08-10T06:35:12,597 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(358): Backup backup_1691649303144 finished: type=FULL,tablelist=ns3:test-16916492264742,targetRootDir=hdfs://localhost:41489/backupUT,startts=1691649307278,completets=1691649312192,bytescopied=0 2023-08-10T06:35:12,597 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-08-10T06:35:12,597 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1691649307464_ns3_test-16916492264742 2023-08-10T06:35:12,598 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1691649307464_ns3_test-16916492264742" 2023-08-10T06:35:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1691649307464_ns3_test-16916492264742 2023-08-10T06:35:12,602 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1691649307464_ns3_test-16916492264742 for backup backup_1691649303144 succeeded. 2023-08-10T06:35:12,602 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:35:12,606 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:35:12,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:35:12,609 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:35:12,610 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:35:12,616 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(376): Backup backup_1691649303144 completed. 2023-08-10T06:35:12,649 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:35:12,653 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:35:12,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649312653 (current time:1691649312653). 2023-08-10T06:35:12,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:35:12,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:35:12,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7a426dd0 to 127.0.0.1:59679 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:35:12,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16048d5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:35:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:35:12,669 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56250, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:35:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7a426dd0 to 127.0.0.1:59679 2023-08-10T06:35:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:35:12,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:35:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure2.ProcedureExecutor(1033): Stored pid=50, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:35:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 50 2023-08-10T06:35:12,674 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:35:12,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-08-10T06:35:12,675 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:35:12,677 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:35:12,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-08-10T06:35:12,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-08-10T06:35:13,083 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:35:13,083 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47}] 2023-08-10T06:35:13,085 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:35:13,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44853 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=51 2023-08-10T06:35:13,237 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. 2023-08-10T06:35:13,237 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2819): Flushing 60ed3292def93d2ecd5862b25318ed47 2/2 column families, dataSize=1.14 KB heapSize=2.23 KB 2023-08-10T06:35:13,242 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/meta/d8d4cae980914e1ea84eb5038f273201 is 171, key is trslm:hdfs://localhost:41489/backupUT\x00ns3:test-16916492264742/meta:log-roll-map/1691649312189/Put/seqid=0 2023-08-10T06:35:13,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-08-10T06:35:13,650 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=458 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/meta/d8d4cae980914e1ea84eb5038f273201 2023-08-10T06:35:13,656 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: d8d4cae980914e1ea84eb5038f273201 2023-08-10T06:35:13,659 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/session/e87b306bd0fb4030ac6059a43fd7e96b is 446, key is session:backup_1691649303144/session:context/1691649312609/Put/seqid=0 2023-08-10T06:35:13,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-08-10T06:35:14,063 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=712 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/session/e87b306bd0fb4030ac6059a43fd7e96b 2023-08-10T06:35:14,069 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e87b306bd0fb4030ac6059a43fd7e96b 2023-08-10T06:35:14,069 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/meta/d8d4cae980914e1ea84eb5038f273201 as hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/d8d4cae980914e1ea84eb5038f273201 2023-08-10T06:35:14,076 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/d8d4cae980914e1ea84eb5038f273201, entries=4, sequenceid=72, filesize=5.6 K 2023-08-10T06:35:14,077 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/session/e87b306bd0fb4030ac6059a43fd7e96b as hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/e87b306bd0fb4030ac6059a43fd7e96b 2023-08-10T06:35:14,083 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/e87b306bd0fb4030ac6059a43fd7e96b, entries=2, sequenceid=72, filesize=5.4 K 2023-08-10T06:35:14,084 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1170, heapSize ~2.20 KB/2248, currentSize=0 B/0 for 60ed3292def93d2ecd5862b25318ed47 in 847ms, sequenceid=72, compaction requested=true 2023-08-10T06:35:14,084 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2520): Flush status journal for 60ed3292def93d2ecd5862b25318ed47: 2023-08-10T06:35:14,084 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. for snapshot_backup_system completed. 2023-08-10T06:35:14,084 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:35:14,084 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:35:14,084 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/ee5a7bc3977042ada997c293856052bb, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/d4291c086d8c4791bd00119e10f93814, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/b7540cf2901649cc88f83d61ac6d6a1f, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/20be7fbcf544464eaf03f133b5735605, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/10815475db68465796bf1f8cdb79c442, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/d8d4cae980914e1ea84eb5038f273201] hfiles 2023-08-10T06:35:14,084 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (1/6): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/ee5a7bc3977042ada997c293856052bb for snapshot=snapshot_backup_system 2023-08-10T06:35:14,084 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (2/6): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/d4291c086d8c4791bd00119e10f93814 for snapshot=snapshot_backup_system 2023-08-10T06:35:14,084 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (3/6): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/b7540cf2901649cc88f83d61ac6d6a1f for snapshot=snapshot_backup_system 2023-08-10T06:35:14,084 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (4/6): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/20be7fbcf544464eaf03f133b5735605 for snapshot=snapshot_backup_system 2023-08-10T06:35:14,084 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (5/6): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/10815475db68465796bf1f8cdb79c442 for snapshot=snapshot_backup_system 2023-08-10T06:35:14,084 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (6/6): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/d8d4cae980914e1ea84eb5038f273201 for snapshot=snapshot_backup_system 2023-08-10T06:35:14,084 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/6743ea07cad64eb59ec77e7feafa1733, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/0eca886709dc4bc28ce13c5c7fbf8954, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/a9dfa7ceabf344a68958248aee65d7ab, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/f9a52bd725e64f909af23b5393aacba4, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/4b4b67835f074c859b336180b4bd2fe3, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/a642f64893574ae7b70f03df104c31d7, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/e87b306bd0fb4030ac6059a43fd7e96b] hfiles 2023-08-10T06:35:14,084 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (1/7): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/6743ea07cad64eb59ec77e7feafa1733 for snapshot=snapshot_backup_system 2023-08-10T06:35:14,084 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (2/7): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/0eca886709dc4bc28ce13c5c7fbf8954 for snapshot=snapshot_backup_system 2023-08-10T06:35:14,084 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (3/7): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/a9dfa7ceabf344a68958248aee65d7ab for snapshot=snapshot_backup_system 2023-08-10T06:35:14,084 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (4/7): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/f9a52bd725e64f909af23b5393aacba4 for snapshot=snapshot_backup_system 2023-08-10T06:35:14,084 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (5/7): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/4b4b67835f074c859b336180b4bd2fe3 for snapshot=snapshot_backup_system 2023-08-10T06:35:14,084 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (6/7): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/a642f64893574ae7b70f03df104c31d7 for snapshot=snapshot_backup_system 2023-08-10T06:35:14,085 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (7/7): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/e87b306bd0fb4030ac6059a43fd7e96b for snapshot=snapshot_backup_system 2023-08-10T06:35:14,490 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. 2023-08-10T06:35:14,490 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=51 2023-08-10T06:35:14,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.HMaster(3994): Remote procedure done, pid=51 2023-08-10T06:35:14,491 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:35:14,492 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:35:14,494 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=51, resume processing ppid=50 2023-08-10T06:35:14,494 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:35:14,494 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=51, ppid=50, state=SUCCESS; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47 in 1.4100 sec 2023-08-10T06:35:14,494 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:35:14,495 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:35:14,495 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:35:14,496 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:35:14,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-08-10T06:35:14,905 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:35:14,917 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:35:14,918 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:35:15,737 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:35:15,737 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 50 2023-08-10T06:35:15,738 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=50, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0660 sec 2023-08-10T06:35:16,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-08-10T06:35:16,782 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 50 completed 2023-08-10T06:35:16,782 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(115): Backup backup_1691649312626 started at 1691649316782. 2023-08-10T06:35:16,783 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(119): Backup session backup_1691649312626 has been started. 2023-08-10T06:35:16,789 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalTableBackupClient(268): For incremental backup, current table set is [test-1691649226474, ns2:test-16916492264741, ns3:test-16916492264742] 2023-08-10T06:35:16,793 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(76): StartCode 1691649284361for backupID backup_1691649312626 2023-08-10T06:35:16,793 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-08-10T06:35:16,794 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-08-10T06:35:16,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-08-10T06:35:16,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-08-10T06:35:16,794 INFO [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-08-10T06:35:16,794 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-08-10T06:35:16,795 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-08-10T06:35:16,795 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-08-10T06:35:16,795 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-08-10T06:35:16,796 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:35:16,797 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-08-10T06:35:16,797 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:16,797 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:35:16,797 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:35:16,797 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:16,797 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-08-10T06:35:16,797 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-08-10T06:35:16,798 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-08-10T06:35:16,798 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-08-10T06:35:16,798 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:35:16,798 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-08-10T06:35:16,798 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-08-10T06:35:16,798 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-08-10T06:35:16,798 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-08-10T06:35:16,798 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-08-10T06:35:16,799 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-08-10T06:35:16,799 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-08-10T06:35:16,799 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-08-10T06:35:16,799 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,44853,1691649217899' joining acquired barrier for procedure (rolllog) in zk 2023-08-10T06:35:16,802 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:16,802 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-08-10T06:35:16,802 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:16,802 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:35:16,803 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:35:16,803 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-08-10T06:35:16,803 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-08-10T06:35:16,803 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:35:16,803 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:35:16,803 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:35:16,804 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:16,804 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:35:16,804 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,44853,1691649217899' joining acquired barrier for procedure 'rolllog' on coordinator 2023-08-10T06:35:16,805 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@127fec3e[Count = 0] remaining members to acquire global barrier 2023-08-10T06:35:16,805 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-08-10T06:35:16,805 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-08-10T06:35:16,806 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:35:16,806 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-08-10T06:35:16,806 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-08-10T06:35:16,806 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-08-10T06:35:16,806 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:16,806 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-08-10T06:35:16,806 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-08-10T06:35:16,806 DEBUG [rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:16,806 INFO [rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1691649307316 highest: 1691649307316 on jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:16,806 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta:.meta(num 1691649307298) roll requested 2023-08-10T06:35:16,818 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649316806.meta, exclude list is [], retry=0 2023-08-10T06:35:16,822 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK] 2023-08-10T06:35:16,824 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649307298.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649316806.meta 2023-08-10T06:35:16,824 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK]] 2023-08-10T06:35:16,824 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649307298.meta is not closed yet, will try archiving it next time 2023-08-10T06:35:16,824 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44853%2C1691649217899:(num 1691649307316) roll requested 2023-08-10T06:35:16,826 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649307298.meta to hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649307298.meta 2023-08-10T06:35:16,836 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649316824, exclude list is [], retry=0 2023-08-10T06:35:16,839 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK] 2023-08-10T06:35:16,840 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649307316 with entries=10, filesize=2.37 KB; new WAL /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649316824 2023-08-10T06:35:16,840 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38797,DS-21ff02e6-aa2f-4fe1-b567-23041c4322d5,DISK]] 2023-08-10T06:35:16,840 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649307316 is not closed yet, will try archiving it next time 2023-08-10T06:35:16,841 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649295919 to hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649295919 2023-08-10T06:35:16,846 DEBUG [rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-08-10T06:35:16,847 INFO [rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1691649316824 on jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:16,855 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-08-10T06:35:16,855 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,44853,1691649217899' in zk 2023-08-10T06:35:16,857 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:16,857 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-08-10T06:35:16,857 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:16,857 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:35:16,857 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:35:16,857 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:35:16,857 DEBUG [member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-08-10T06:35:16,858 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:35:16,859 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:35:16,859 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:35:16,859 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:16,859 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:35:16,860 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:35:16,860 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:16,860 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,44853,1691649217899': 2023-08-10T06:35:16,861 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,44853,1691649217899' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-08-10T06:35:16,861 INFO [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-08-10T06:35:16,861 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-08-10T06:35:16,861 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-08-10T06:35:16,861 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-08-10T06:35:16,861 INFO [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-08-10T06:35:16,862 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:35:16,862 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:35:16,863 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-08-10T06:35:16,863 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-08-10T06:35:16,863 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:35:16,863 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-08-10T06:35:16,863 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:35:16,863 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:35:16,863 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:35:16,863 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:35:16,863 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:35:16,864 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:16,864 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:35:16,864 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:35:16,868 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:35:16,868 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:35:16,869 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:16,869 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:35:16,869 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:16,869 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:35:16,870 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:16,874 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:16,874 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-08-10T06:35:16,874 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-08-10T06:35:16,875 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-08-10T06:35:16,875 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-08-10T06:35:16,875 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:16,875 DEBUG [(jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:35:16,875 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-08-10T06:35:16,875 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:35:16,875 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:35:16,875 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-08-10T06:35:16,875 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:35:16,875 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:35:16,875 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:35:16,875 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin(2751): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-08-10T06:35:16,876 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin(2759): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-08-10T06:35:16,876 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:35:16,876 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:35:16,976 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin(2765): Getting current status of procedure from master... 2023-08-10T06:35:16,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-08-10T06:35:16,979 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:44853=1691649284361} newestTimestamps: {jenkins-hbase3.apache.org:44853=1691649307316} 2023-08-10T06:35:16,981 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649220360.meta 2023-08-10T06:35:16,981 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649316806.meta 2023-08-10T06:35:16,981 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649307316 2023-08-10T06:35:16,981 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649316824 2023-08-10T06:35:16,982 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649242397.meta 2023-08-10T06:35:16,982 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649261946.meta 2023-08-10T06:35:16,982 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649273234.meta 2023-08-10T06:35:16,982 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649284343.meta 2023-08-10T06:35:16,982 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649295902.meta 2023-08-10T06:35:16,983 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs/jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta.1691649307298.meta 2023-08-10T06:35:16,993 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(146): Attempting to copy table info for:ns2:test-16916492264741 target: hdfs://localhost:41489/backupUT/backup_1691649312626/ns2/test-16916492264741 descriptor: 'ns2:test-16916492264741', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:35:16,993 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-08-10T06:35:16,995 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(151): Starting to write region info for table ns2:test-16916492264741 2023-08-10T06:35:17,401 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(158): Finished writing region info for table ns2:test-16916492264741 2023-08-10T06:35:17,811 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(146): Attempting to copy table info for:ns3:test-16916492264742 target: hdfs://localhost:41489/backupUT/backup_1691649312626/ns3/test-16916492264742 descriptor: 'ns3:test-16916492264742', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:35:17,811 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-08-10T06:35:17,813 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(151): Starting to write region info for table ns3:test-16916492264742 2023-08-10T06:35:18,218 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(158): Finished writing region info for table ns3:test-16916492264742 2023-08-10T06:35:18,627 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(146): Attempting to copy table info for:test-1691649226474 target: hdfs://localhost:41489/backupUT/backup_1691649312626/default/test-1691649226474 descriptor: 'test-1691649226474', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:35:18,627 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-08-10T06:35:18,629 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(151): Starting to write region info for table test-1691649226474 2023-08-10T06:35:19,034 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(158): Finished writing region info for table test-1691649226474 2023-08-10T06:35:19,040 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:41489/backupUT/.tmp/backup_1691649312626 from hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649307316;hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649295919 2023-08-10T06:35:19,040 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0e400d8e to 127.0.0.1:59679 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:35:19,046 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5820d674, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:35:19,047 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:35:19,048 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:49734, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:35:19,051 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-08-10T06:35:19,051 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1691649226474 2023-08-10T06:35:19,054 DEBUG [hconnection-0x2ccdbc6e-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:35:19,056 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56258, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:35:19,057 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1691649226474: test-1691649226474; 2023-08-10T06:35:19,057 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16916492264741 2023-08-10T06:35:19,058 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16916492264741: ns2:test-16916492264741; 2023-08-10T06:35:19,058 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns3:test-16916492264742 2023-08-10T06:35:19,060 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns3:test-16916492264742: ns3:test-16916492264742; 2023-08-10T06:35:19,060 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(694): Configuring 3 reduce partitions to match current region count for all tables 2023-08-10T06:35:19,060 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/hbase-staging/partitions_6447f5e6-2c6f-401c-b7fa-cba9c0f58f26 2023-08-10T06:35:19,467 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:19,467 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:19,468 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:19,468 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:20,459 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/hadoop-278591443430644800.jar 2023-08-10T06:35:20,459 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:20,460 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:20,460 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:20,461 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:20,461 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:20,461 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:20,462 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:20,462 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:20,462 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:20,463 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:35:20,464 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:35:20,464 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:35:20,464 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:35:20,465 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:35:20,465 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:35:20,466 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:35:20,466 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:35:20,466 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:35:20,467 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:35:20,467 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:35:20,468 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:35:20,468 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:20,469 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:20,469 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:20,469 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:20,469 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:20,470 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:35:20,470 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:35:20,470 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: default:test-1691649226474,ns2:test-16916492264741,ns3:test-16916492264742 2023-08-10T06:35:20,471 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] client.ConnectionImplementation(2114): Closing master protocol: MasterService 2023-08-10T06:35:20,471 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0e400d8e to 127.0.0.1:59679 2023-08-10T06:35:20,471 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:35:20,472 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:35:21,429 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/hadoop-364955814145350717.jar 2023-08-10T06:35:21,437 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-08-10T06:35:21,451 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649307316 for WAL files 2023-08-10T06:35:21,452 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649307316 2023-08-10T06:35:21,452 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649295919 for WAL files 2023-08-10T06:35:21,453 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649295919 2023-08-10T06:35:21,523 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/mapreduce.cluster.local.dir/1691649321491/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:35:21,524 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/mapreduce.cluster.local.dir/1691649321491/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:35:21,575 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649295919 for hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649295919 (-9223372036854775808:9223372036854775807) length:2574 2023-08-10T06:35:21,578 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-08-10T06:35:21,578 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-08-10T06:35:21,626 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649307316 for hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649307316 (-9223372036854775808:9223372036854775807) length:2432 2023-08-10T06:35:21,629 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-08-10T06:35:21,629 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-08-10T06:35:22,559 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:41489/backupUT 2023-08-10T06:35:22,559 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1691649312626 2023-08-10T06:35:22,559 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-08-10T06:35:22,575 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:41489/backupUT/.tmp/backup_1691649312626, hdfs://localhost:41489/backupUT] 2023-08-10T06:35:22,599 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-08-10T06:35:23,189 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-08-10T06:35:23,191 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1691649312626 2023-08-10T06:35:23,191 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 0 bytes copied." 2023-08-10T06:35:23,191 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local1212874633_0013 completed: true true 2023-08-10T06:35:23,194 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=6749955 FILE: Number of bytes written=15998155 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=335295 HDFS: Number of bytes written=2484406 HDFS: Number of read operations=1146 HDFS: Number of large read operations=0 HDFS: Number of write operations=431 Map-Reduce Framework Map input records=2 Map output records=0 Input split bytes=341 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1535115264 File Input Format Counters Bytes Read=486 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=0 Bytes Expected=0 Files Copied=1 DIR_COPY=1 2023-08-10T06:35:23,194 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:41489/backupUT/.tmp/backup_1691649312626 to hdfs://localhost:41489/backupUT finished. 2023-08-10T06:35:23,203 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(433): deleted 0 original bulkload rows 2023-08-10T06:35:23,203 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649312626 2023-08-10T06:35:23,207 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649291725 to construct the dependency. 2023-08-10T06:35:23,207 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649291725 2023-08-10T06:35:23,210 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649291725/.backup.manifest 2023-08-10T06:35:23,210 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649291725BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:23,210 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649280173 to construct the dependency. 2023-08-10T06:35:23,210 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649280173 2023-08-10T06:35:23,212 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649280173/.backup.manifest 2023-08-10T06:35:23,212 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649280173BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:23,212 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649269054 to construct the dependency. 2023-08-10T06:35:23,212 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649269054 2023-08-10T06:35:23,214 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649269054/.backup.manifest 2023-08-10T06:35:23,214 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649269054BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:23,214 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649257759 to construct the dependency. 2023-08-10T06:35:23,214 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649257759 2023-08-10T06:35:23,216 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649257759/.backup.manifest 2023-08-10T06:35:23,216 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649257759BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:23,216 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(340): Got 6 ancestors for the current backup. 2023-08-10T06:35:23,216 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649312626 2023-08-10T06:35:23,218 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649291725 to construct the dependency. 2023-08-10T06:35:23,218 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649291725 2023-08-10T06:35:23,220 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649291725/.backup.manifest 2023-08-10T06:35:23,220 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649291725BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:23,220 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649280173 to construct the dependency. 2023-08-10T06:35:23,220 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649280173 2023-08-10T06:35:23,222 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649280173/.backup.manifest 2023-08-10T06:35:23,222 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649280173BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:23,222 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649269054 to construct the dependency. 2023-08-10T06:35:23,222 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649269054 2023-08-10T06:35:23,224 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649269054/.backup.manifest 2023-08-10T06:35:23,224 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649269054BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:23,224 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649257759 to construct the dependency. 2023-08-10T06:35:23,224 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649257759 2023-08-10T06:35:23,225 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649257759/.backup.manifest 2023-08-10T06:35:23,226 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649257759BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:23,226 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(340): Got 6 ancestors for the current backup. 2023-08-10T06:35:23,631 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:41489/backupUT/backup_1691649312626/.backup.manifest 2023-08-10T06:35:23,632 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649312626 2023-08-10T06:35:23,634 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649291725 to construct the dependency. 2023-08-10T06:35:23,634 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649291725 2023-08-10T06:35:23,638 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649291725/.backup.manifest 2023-08-10T06:35:23,638 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649291725BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:23,638 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649280173 to construct the dependency. 2023-08-10T06:35:23,638 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649280173 2023-08-10T06:35:23,641 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649280173/.backup.manifest 2023-08-10T06:35:23,641 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649280173BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:23,641 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649269054 to construct the dependency. 2023-08-10T06:35:23,641 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649269054 2023-08-10T06:35:23,643 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649269054/.backup.manifest 2023-08-10T06:35:23,643 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649269054BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:23,643 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649257759 to construct the dependency. 2023-08-10T06:35:23,643 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649257759 2023-08-10T06:35:23,645 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649257759/.backup.manifest 2023-08-10T06:35:23,645 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649257759BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:23,645 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(340): Got 6 ancestors for the current backup. 2023-08-10T06:35:23,645 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649312626 2023-08-10T06:35:23,648 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649291725 to construct the dependency. 2023-08-10T06:35:23,648 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649291725 2023-08-10T06:35:23,650 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649291725/.backup.manifest 2023-08-10T06:35:23,650 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649291725BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:23,650 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649280173 to construct the dependency. 2023-08-10T06:35:23,650 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649280173 2023-08-10T06:35:23,652 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649280173/.backup.manifest 2023-08-10T06:35:23,652 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649280173BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:23,652 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649269054 to construct the dependency. 2023-08-10T06:35:23,653 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649269054 2023-08-10T06:35:23,655 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649269054/.backup.manifest 2023-08-10T06:35:23,655 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649269054BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:23,655 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649257759 to construct the dependency. 2023-08-10T06:35:23,655 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649257759 2023-08-10T06:35:23,657 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649257759/.backup.manifest 2023-08-10T06:35:23,657 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649257759BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:23,657 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(340): Got 6 ancestors for the current backup. 2023-08-10T06:35:24,063 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:41489/backupUT/backup_1691649312626/.backup.manifest 2023-08-10T06:35:24,063 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649312626 2023-08-10T06:35:24,065 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649291725 to construct the dependency. 2023-08-10T06:35:24,065 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649291725 2023-08-10T06:35:24,068 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649291725/.backup.manifest 2023-08-10T06:35:24,068 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649291725BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:24,068 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649280173 to construct the dependency. 2023-08-10T06:35:24,068 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649280173 2023-08-10T06:35:24,070 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649280173/.backup.manifest 2023-08-10T06:35:24,070 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649280173BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:24,070 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649269054 to construct the dependency. 2023-08-10T06:35:24,070 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649269054 2023-08-10T06:35:24,072 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649269054/.backup.manifest 2023-08-10T06:35:24,072 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649269054BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:24,072 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649257759 to construct the dependency. 2023-08-10T06:35:24,072 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649257759 2023-08-10T06:35:24,074 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649257759/.backup.manifest 2023-08-10T06:35:24,074 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649257759BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:24,074 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(340): Got 6 ancestors for the current backup. 2023-08-10T06:35:24,074 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649312626 2023-08-10T06:35:24,076 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649291725 to construct the dependency. 2023-08-10T06:35:24,076 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649291725 2023-08-10T06:35:24,078 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649291725/.backup.manifest 2023-08-10T06:35:24,078 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649291725BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:24,078 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649280173 to construct the dependency. 2023-08-10T06:35:24,078 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649280173 2023-08-10T06:35:24,080 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649280173/.backup.manifest 2023-08-10T06:35:24,080 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649280173BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:24,080 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649269054 to construct the dependency. 2023-08-10T06:35:24,080 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649269054 2023-08-10T06:35:24,082 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649269054/.backup.manifest 2023-08-10T06:35:24,082 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649269054BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:24,082 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649257759 to construct the dependency. 2023-08-10T06:35:24,082 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649257759 2023-08-10T06:35:24,083 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649257759/.backup.manifest 2023-08-10T06:35:24,083 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649257759BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:24,083 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(340): Got 6 ancestors for the current backup. 2023-08-10T06:35:24,489 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:41489/backupUT/backup_1691649312626/.backup.manifest 2023-08-10T06:35:24,489 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649312626 2023-08-10T06:35:24,491 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649291725 to construct the dependency. 2023-08-10T06:35:24,491 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649291725 2023-08-10T06:35:24,494 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649291725/.backup.manifest 2023-08-10T06:35:24,494 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649291725BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:24,494 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649280173 to construct the dependency. 2023-08-10T06:35:24,494 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649280173 2023-08-10T06:35:24,496 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649280173/.backup.manifest 2023-08-10T06:35:24,496 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649280173BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:24,496 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649269054 to construct the dependency. 2023-08-10T06:35:24,496 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649269054 2023-08-10T06:35:24,498 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649269054/.backup.manifest 2023-08-10T06:35:24,498 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649269054BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:24,498 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:41489/backupUT/backup_1691649257759 to construct the dependency. 2023-08-10T06:35:24,499 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:41489/backupUT/backup_1691649257759 2023-08-10T06:35:24,500 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1691649257759/.backup.manifest 2023-08-10T06:35:24,500 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1691649257759BackupDir=hdfs://localhost:41489/backupUT} 2023-08-10T06:35:24,500 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManager(340): Got 6 ancestors for the current backup. 2023-08-10T06:35:24,906 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:41489/backupUT/backup_1691649312626/.backup.manifest 2023-08-10T06:35:24,906 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(358): Backup backup_1691649312626 finished: type=INCREMENTAL,tablelist=ns2:test-16916492264741;ns3:test-16916492264742;test-1691649226474,targetRootDir=hdfs://localhost:41489/backupUT,startts=1691649316782,completets=1691649323203,bytescopied=0 2023-08-10T06:35:24,906 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:35:24,911 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:35:24,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:35:24,914 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:35:24,915 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:35:24,918 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.TableBackupClient(376): Backup backup_1691649312626 completed. 2023-08-10T06:35:24,927 ERROR [Listener at jenkins-hbase3.apache.org/36733 {}] backup.TestBackupMultipleDeletes(130): Delete backupIdInc2 2023-08-10T06:35:24,934 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:35:24,941 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:35:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649324941 (current time:1691649324941). 2023-08-10T06:35:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:35:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:35:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1c7c3434 to 127.0.0.1:59679 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:35:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b688b7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:35:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:35:24,951 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:35172, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:35:24,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1c7c3434 to 127.0.0.1:59679 2023-08-10T06:35:24,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:35:24,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:35:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure2.ProcedureExecutor(1033): Stored pid=52, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:35:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-08-10T06:35:24,954 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:35:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-08-10T06:35:24,955 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:35:24,956 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:35:25,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-08-10T06:35:25,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-08-10T06:35:25,362 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:35:25,362 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47}] 2023-08-10T06:35:25,364 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:35:25,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44853 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=53 2023-08-10T06:35:25,515 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. 2023-08-10T06:35:25,516 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2819): Flushing 60ed3292def93d2ecd5862b25318ed47 2/2 column families, dataSize=1.75 KB heapSize=3.05 KB 2023-08-10T06:35:25,519 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/meta/055007204d8d4f4b8fccb41d0b6ece3c is 171, key is trslm:hdfs://localhost:41489/backupUT\x00ns2:test-16916492264741/meta:log-roll-map/1691649323198/Put/seqid=0 2023-08-10T06:35:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-08-10T06:35:25,924 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=758 B at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/meta/055007204d8d4f4b8fccb41d0b6ece3c 2023-08-10T06:35:25,928 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 055007204d8d4f4b8fccb41d0b6ece3c 2023-08-10T06:35:25,931 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/session/90c628ec035c4038870ae2250993469d is 461, key is session:backup_1691649312626/session:context/1691649324914/Put/seqid=0 2023-08-10T06:35:26,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-08-10T06:35:26,335 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.01 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/session/90c628ec035c4038870ae2250993469d 2023-08-10T06:35:26,340 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 90c628ec035c4038870ae2250993469d 2023-08-10T06:35:26,341 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/meta/055007204d8d4f4b8fccb41d0b6ece3c as hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/055007204d8d4f4b8fccb41d0b6ece3c 2023-08-10T06:35:26,346 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/055007204d8d4f4b8fccb41d0b6ece3c, entries=6, sequenceid=84, filesize=5.8 K 2023-08-10T06:35:26,347 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/session/90c628ec035c4038870ae2250993469d as hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/90c628ec035c4038870ae2250993469d 2023-08-10T06:35:26,351 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/90c628ec035c4038870ae2250993469d, entries=2, sequenceid=84, filesize=5.4 K 2023-08-10T06:35:26,352 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(3022): Finished flush of dataSize ~1.75 KB/1792, heapSize ~3.02 KB/3096, currentSize=0 B/0 for 60ed3292def93d2ecd5862b25318ed47 in 837ms, sequenceid=84, compaction requested=true 2023-08-10T06:35:26,352 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2520): Flush status journal for 60ed3292def93d2ecd5862b25318ed47: 2023-08-10T06:35:26,352 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. for snapshot_backup_system completed. 2023-08-10T06:35:26,352 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:35:26,352 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:35:26,352 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/ee5a7bc3977042ada997c293856052bb, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/d4291c086d8c4791bd00119e10f93814, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/b7540cf2901649cc88f83d61ac6d6a1f, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/20be7fbcf544464eaf03f133b5735605, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/10815475db68465796bf1f8cdb79c442, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/d8d4cae980914e1ea84eb5038f273201, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/055007204d8d4f4b8fccb41d0b6ece3c] hfiles 2023-08-10T06:35:26,352 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (1/7): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/ee5a7bc3977042ada997c293856052bb for snapshot=snapshot_backup_system 2023-08-10T06:35:26,352 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (2/7): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/d4291c086d8c4791bd00119e10f93814 for snapshot=snapshot_backup_system 2023-08-10T06:35:26,352 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (3/7): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/b7540cf2901649cc88f83d61ac6d6a1f for snapshot=snapshot_backup_system 2023-08-10T06:35:26,352 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (4/7): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/20be7fbcf544464eaf03f133b5735605 for snapshot=snapshot_backup_system 2023-08-10T06:35:26,352 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (5/7): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/10815475db68465796bf1f8cdb79c442 for snapshot=snapshot_backup_system 2023-08-10T06:35:26,353 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (6/7): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/d8d4cae980914e1ea84eb5038f273201 for snapshot=snapshot_backup_system 2023-08-10T06:35:26,353 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (7/7): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/055007204d8d4f4b8fccb41d0b6ece3c for snapshot=snapshot_backup_system 2023-08-10T06:35:26,353 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/6743ea07cad64eb59ec77e7feafa1733, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/0eca886709dc4bc28ce13c5c7fbf8954, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/a9dfa7ceabf344a68958248aee65d7ab, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/f9a52bd725e64f909af23b5393aacba4, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/4b4b67835f074c859b336180b4bd2fe3, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/a642f64893574ae7b70f03df104c31d7, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/e87b306bd0fb4030ac6059a43fd7e96b, hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/90c628ec035c4038870ae2250993469d] hfiles 2023-08-10T06:35:26,353 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (1/8): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/6743ea07cad64eb59ec77e7feafa1733 for snapshot=snapshot_backup_system 2023-08-10T06:35:26,353 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (2/8): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/0eca886709dc4bc28ce13c5c7fbf8954 for snapshot=snapshot_backup_system 2023-08-10T06:35:26,353 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (3/8): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/a9dfa7ceabf344a68958248aee65d7ab for snapshot=snapshot_backup_system 2023-08-10T06:35:26,353 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (4/8): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/f9a52bd725e64f909af23b5393aacba4 for snapshot=snapshot_backup_system 2023-08-10T06:35:26,353 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (5/8): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/4b4b67835f074c859b336180b4bd2fe3 for snapshot=snapshot_backup_system 2023-08-10T06:35:26,353 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (6/8): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/a642f64893574ae7b70f03df104c31d7 for snapshot=snapshot_backup_system 2023-08-10T06:35:26,353 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (7/8): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/e87b306bd0fb4030ac6059a43fd7e96b for snapshot=snapshot_backup_system 2023-08-10T06:35:26,353 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (8/8): hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/90c628ec035c4038870ae2250993469d for snapshot=snapshot_backup_system 2023-08-10T06:35:26,758 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. 2023-08-10T06:35:26,758 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=53 2023-08-10T06:35:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.HMaster(3994): Remote procedure done, pid=53 2023-08-10T06:35:26,759 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:35:26,760 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:35:26,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=53, resume processing ppid=52 2023-08-10T06:35:26,762 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:35:26,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=53, ppid=52, state=SUCCESS; SnapshotRegionProcedure 60ed3292def93d2ecd5862b25318ed47 in 1.3990 sec 2023-08-10T06:35:26,763 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:35:26,763 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:35:26,763 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:35:26,764 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:35:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-08-10T06:35:27,175 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:35:27,187 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:35:27,187 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:35:28,005 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:35:28,005 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-08-10T06:35:28,006 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=52, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0530 sec 2023-08-10T06:35:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-08-10T06:35:29,061 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 52 completed 2023-08-10T06:35:29,066 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupAdminImpl(227): Deleting backup backup_1691649269054 ... 2023-08-10T06:35:29,066 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(435): Delete log files: .backup.manifest 2023-08-10T06:35:29,067 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(435): Delete log files: _SUCCESS 2023-08-10T06:35:29,068 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(435): Delete log files: default 2023-08-10T06:35:29,068 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(435): Delete log files: ns2 2023-08-10T06:35:29,069 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(443): Trying to cleanup up target dir : backup_1691649269054 2023-08-10T06:35:29,069 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(458): No data has been found in hdfs://localhost:41489/backupUT/backup_1691649269054/ns2/test-16916492264741. 2023-08-10T06:35:29,070 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(465): hdfs://localhost:41489/backupUT/backup_1691649269054/ns2 is empty, remove it. 2023-08-10T06:35:29,070 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(458): No data has been found in hdfs://localhost:41489/backupUT/backup_1691649269054/default/test-1691649226474. 2023-08-10T06:35:29,070 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(465): hdfs://localhost:41489/backupUT/backup_1691649269054/default is empty, remove it. 2023-08-10T06:35:29,074 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupAdminImpl(309): GetAffectedBackupInfos for: backup_1691649269054 table=ns2:test-16916492264741 2023-08-10T06:35:29,075 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupAdminImpl(326): GetAffectedBackupInfos for: backup_1691649269054 table=ns2:test-16916492264741 added backup_1691649312626 tables={ns2:test-16916492264741,ns3:test-16916492264742,test-1691649226474} 2023-08-10T06:35:29,075 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupAdminImpl(326): GetAffectedBackupInfos for: backup_1691649269054 table=ns2:test-16916492264741 added backup_1691649291725 tables={ns2:test-16916492264741,test-1691649226474} 2023-08-10T06:35:29,076 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupAdminImpl(326): GetAffectedBackupInfos for: backup_1691649269054 table=ns2:test-16916492264741 added backup_1691649280173 tables={ns2:test-16916492264741,test-1691649226474} 2023-08-10T06:35:29,076 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupAdminImpl(287): Remove ns2:test-16916492264741 from backup_1691649312626 tables={ns2:test-16916492264741,ns3:test-16916492264742,test-1691649226474} 2023-08-10T06:35:29,078 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupAdminImpl(354): Cleaning up backup data at hdfs://localhost:41489/backupUT/backup_1691649312626/ns2/test-16916492264741 done. 2023-08-10T06:35:29,078 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupAdminImpl(287): Remove ns2:test-16916492264741 from backup_1691649291725 tables={ns2:test-16916492264741,test-1691649226474} 2023-08-10T06:35:29,079 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupAdminImpl(354): Cleaning up backup data at hdfs://localhost:41489/backupUT/backup_1691649291725/ns2/test-16916492264741 done. 2023-08-10T06:35:29,080 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupAdminImpl(287): Remove ns2:test-16916492264741 from backup_1691649280173 tables={ns2:test-16916492264741,test-1691649226474} 2023-08-10T06:35:29,081 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupAdminImpl(354): Cleaning up backup data at hdfs://localhost:41489/backupUT/backup_1691649280173/ns2/test-16916492264741 done. 2023-08-10T06:35:29,083 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupAdminImpl(309): GetAffectedBackupInfos for: backup_1691649269054 table=test-1691649226474 2023-08-10T06:35:29,085 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupAdminImpl(326): GetAffectedBackupInfos for: backup_1691649269054 table=test-1691649226474 added backup_1691649312626 tables={ns3:test-16916492264742,test-1691649226474} 2023-08-10T06:35:29,085 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupAdminImpl(326): GetAffectedBackupInfos for: backup_1691649269054 table=test-1691649226474 added backup_1691649291725 tables={test-1691649226474} 2023-08-10T06:35:29,085 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupAdminImpl(326): GetAffectedBackupInfos for: backup_1691649269054 table=test-1691649226474 added backup_1691649280173 tables={test-1691649226474} 2023-08-10T06:35:29,085 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupAdminImpl(287): Remove test-1691649226474 from backup_1691649312626 tables={ns3:test-16916492264742,test-1691649226474} 2023-08-10T06:35:29,087 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupAdminImpl(354): Cleaning up backup data at hdfs://localhost:41489/backupUT/backup_1691649312626/default/test-1691649226474 done. 2023-08-10T06:35:29,087 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupAdminImpl(287): Remove test-1691649226474 from backup_1691649291725 tables={test-1691649226474} 2023-08-10T06:35:29,087 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupAdminImpl(293): Delete backup info backup_1691649291725 2023-08-10T06:35:29,090 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(435): Delete log files: .backup.manifest 2023-08-10T06:35:29,090 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(435): Delete log files: _SUCCESS 2023-08-10T06:35:29,091 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(435): Delete log files: default 2023-08-10T06:35:29,091 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(435): Delete log files: ns2 2023-08-10T06:35:29,092 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(443): Trying to cleanup up target dir : backup_1691649291725 2023-08-10T06:35:29,092 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(458): No data has been found in hdfs://localhost:41489/backupUT/backup_1691649291725/default/test-1691649226474. 2023-08-10T06:35:29,093 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(465): hdfs://localhost:41489/backupUT/backup_1691649291725/default is empty, remove it. 2023-08-10T06:35:29,093 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupAdminImpl(287): Remove test-1691649226474 from backup_1691649280173 tables={test-1691649226474} 2023-08-10T06:35:29,093 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupAdminImpl(293): Delete backup info backup_1691649280173 2023-08-10T06:35:29,095 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(435): Delete log files: .backup.manifest 2023-08-10T06:35:29,095 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(435): Delete log files: _SUCCESS 2023-08-10T06:35:29,095 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(435): Delete log files: default 2023-08-10T06:35:29,096 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(435): Delete log files: ns2 2023-08-10T06:35:29,096 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(443): Trying to cleanup up target dir : backup_1691649280173 2023-08-10T06:35:29,097 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(458): No data has been found in hdfs://localhost:41489/backupUT/backup_1691649280173/default/test-1691649226474. 2023-08-10T06:35:29,097 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.BackupUtils(465): hdfs://localhost:41489/backupUT/backup_1691649280173/default is empty, remove it. 2023-08-10T06:35:29,099 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupAdminImpl(269): 0 bulk loaded files out of 0 were deleted 2023-08-10T06:35:29,099 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(433): deleted 0 original bulkload rows 2023-08-10T06:35:29,100 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupAdminImpl(276): Delete backup backup_1691649269054 completed. 2023-08-10T06:35:29,104 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(1014): test-1691649226474 2023-08-10T06:35:29,104 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(1014): ns2:test-16916492264741 2023-08-10T06:35:29,104 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(1014): ns3:test-16916492264742 2023-08-10T06:35:29,106 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:35:29,110 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:35:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:35:29,112 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:35:29,112 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:35:29,114 ERROR [Listener at jenkins-hbase3.apache.org/36733 {}] backup.TestBackupMultipleDeletes(132): Delete backupIdInc2 done 2023-08-10T06:35:29,124 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] client.ConnectionImplementation(2114): Closing master protocol: MasterService 2023-08-10T06:35:29,124 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x22204ed5 to 127.0.0.1:59679 2023-08-10T06:35:29,125 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:35:29,146 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] hbase.ResourceChecker(175): after: backup.TestBackupMultipleDeletes#testBackupMultipleDeletes Thread=590 (was 582) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-762059543_16 at /127.0.0.1:36846 [Waiting for operation #3] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-548592812-172.31.12.81-1691649211357:blk_1073741979_1155, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-332398480_16 at /127.0.0.1:36864 [Waiting for operation #11] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x7a45b7aa-shared-pool-31 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/cluster_727a6324-ee3a-516e-29f4-b0167fb1d918/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2044625702_16 at /127.0.0.1:59316 [Receiving block BP-548592812-172.31.12.81-1691649211357:blk_1073741980_1156] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: region-location-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: member: 'jenkins-hbase3.apache.org,44853,1691649217899' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-762059543_16 at /127.0.0.1:36758 [Waiting for operation #45] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/cluster_727a6324-ee3a-516e-29f4-b0167fb1d918/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: (jenkins-hbase3.apache.org,37947,1691649214739)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-548592812-172.31.12.81-1691649211357:blk_1073741980_1156, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2044625702_16 at /127.0.0.1:59302 [Receiving block BP-548592812-172.31.12.81-1691649211357:blk_1073741979_1155] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,44853,1691649217899)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=677 (was 660) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=191 (was 282), ProcessCount=171 (was 171), AvailableMemoryMB=1388 (was 2070) 2023-08-10T06:35:29,147 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] hbase.ResourceChecker(130): Thread=590 is superior to 500 2023-08-10T06:35:29,154 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-08-10T06:35:29,155 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] client.ConnectionImplementation(2114): Closing master protocol: MasterService 2023-08-10T06:35:29,155 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0fbc2dc4 to 127.0.0.1:59679 2023-08-10T06:35:29,155 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:35:29,155 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-08-10T06:35:29,155 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] util.JVMClusterUtil(257): Found active master hash=1230679401, stopped=false 2023-08-10T06:35:29,155 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] master.ServerManager(910): Cluster shutdown requested of master=jenkins-hbase3.apache.org,37947,1691649214739 2023-08-10T06:35:29,158 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2023-08-10T06:35:29,158 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2023-08-10T06:35:29,158 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:35:29,158 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:35:29,158 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] procedure2.ProcedureExecutor(631): Stopping 2023-08-10T06:35:29,159 DEBUG [Listener at jenkins-hbase3.apache.org/36733 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:35:29,159 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] regionserver.HRegionServer(2513): ***** STOPPING region server 'jenkins-hbase3.apache.org,44853,1691649217899' ***** 2023-08-10T06:35:29,159 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-08-10T06:35:29,159 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] regionserver.HRegionServer(2527): STOPPED: Shutdown requested 2023-08-10T06:35:29,159 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-08-10T06:35:29,159 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-08-10T06:35:29,173 INFO [RS:0;jenkins-hbase3:44853 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@343d88aa{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-08-10T06:35:29,177 INFO [RS:0;jenkins-hbase3:44853 {}] server.AbstractConnector(383): Stopped ServerConnector@276d566e{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-08-10T06:35:29,177 INFO [RS:0;jenkins-hbase3:44853 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-08-10T06:35:29,178 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-08-10T06:35:29,178 INFO [RS:0;jenkins-hbase3:44853 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@5452a265{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-08-10T06:35:29,178 INFO [RS:0;jenkins-hbase3:44853 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@5f1c3a05{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/hadoop.log.dir/,STOPPED} 2023-08-10T06:35:29,178 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-08-10T06:35:29,181 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.HeapMemoryManager(220): Stopping 2023-08-10T06:35:29,181 INFO [RS:0;jenkins-hbase3:44853 {}] flush.RegionServerFlushTableProcedureManager(117): Stopping region server flush procedure manager gracefully. 2023-08-10T06:35:29,181 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-08-10T06:35:29,181 INFO [RS:0;jenkins-hbase3:44853 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-08-10T06:35:29,181 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(3524): Received CLOSE for a9e9ef626a68c685245826f3ca7f958f 2023-08-10T06:35:29,182 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(3524): Received CLOSE for 2468e04f6179571f82a0adee7c1e6692 2023-08-10T06:35:29,182 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(3524): Received CLOSE for 2a873d0a0a9423038307a20a4f0e1113 2023-08-10T06:35:29,182 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(3524): Received CLOSE for 930ae4244d4cd65da422c67705de93c0 2023-08-10T06:35:29,182 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(3524): Received CLOSE for 60ed3292def93d2ecd5862b25318ed47 2023-08-10T06:35:29,182 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(3524): Received CLOSE for 3e5d3d5b3b7bc4e6a979a8585e22b81f 2023-08-10T06:35:29,182 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(3524): Received CLOSE for 94f564ab80fb7e8519555fad2ae5463b 2023-08-10T06:35:29,182 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:29,182 INFO [RS:0;jenkins-hbase3:44853 {}] client.ConnectionImplementation(2114): Closing master protocol: MasterService 2023-08-10T06:35:29,182 DEBUG [RS:0;jenkins-hbase3:44853 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:35:29,182 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing a9e9ef626a68c685245826f3ca7f958f, disabling compactions & flushes 2023-08-10T06:35:29,182 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-16916492264742,,1691649230428.a9e9ef626a68c685245826f3ca7f958f. 2023-08-10T06:35:29,182 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-08-10T06:35:29,182 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16916492264742,,1691649230428.a9e9ef626a68c685245826f3ca7f958f. 2023-08-10T06:35:29,182 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16916492264742,,1691649230428.a9e9ef626a68c685245826f3ca7f958f. after waiting 0 ms 2023-08-10T06:35:29,182 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16916492264742,,1691649230428.a9e9ef626a68c685245826f3ca7f958f. 2023-08-10T06:35:29,182 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-08-10T06:35:29,182 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-08-10T06:35:29,182 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(3524): Received CLOSE for 1588230740 2023-08-10T06:35:29,183 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(1579): Waiting on 8 regions to close 2023-08-10T06:35:29,183 DEBUG [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(1583): Online Regions={a9e9ef626a68c685245826f3ca7f958f=ns3:test-16916492264742,,1691649230428.a9e9ef626a68c685245826f3ca7f958f., 2468e04f6179571f82a0adee7c1e6692=backup:system_bulk,,1691649236001.2468e04f6179571f82a0adee7c1e6692., 2a873d0a0a9423038307a20a4f0e1113=hbase:namespace,,1691649220813.2a873d0a0a9423038307a20a4f0e1113., 930ae4244d4cd65da422c67705de93c0=ns4:test-16916492264743,,1691649232552.930ae4244d4cd65da422c67705de93c0., 60ed3292def93d2ecd5862b25318ed47=backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47., 1588230740=hbase:meta,,1.1588230740, 3e5d3d5b3b7bc4e6a979a8585e22b81f=test-1691649226474,,1691649226962.3e5d3d5b3b7bc4e6a979a8585e22b81f., 94f564ab80fb7e8519555fad2ae5463b=ns2:test-16916492264741,,1691649228221.94f564ab80fb7e8519555fad2ae5463b.} 2023-08-10T06:35:29,183 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-08-10T06:35:29,183 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-08-10T06:35:29,184 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-08-10T06:35:29,184 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-08-10T06:35:29,184 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-08-10T06:35:29,186 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=9.25 KB heapSize=16.20 KB 2023-08-10T06:35:29,186 DEBUG [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 2468e04f6179571f82a0adee7c1e6692, 2a873d0a0a9423038307a20a4f0e1113, 3e5d3d5b3b7bc4e6a979a8585e22b81f, 60ed3292def93d2ecd5862b25318ed47, 930ae4244d4cd65da422c67705de93c0, 94f564ab80fb7e8519555fad2ae5463b, a9e9ef626a68c685245826f3ca7f958f 2023-08-10T06:35:29,189 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/ns3/test-16916492264742/a9e9ef626a68c685245826f3ca7f958f/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2023-08-10T06:35:29,190 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:35:29,191 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-16916492264742,,1691649230428.a9e9ef626a68c685245826f3ca7f958f. 2023-08-10T06:35:29,191 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for a9e9ef626a68c685245826f3ca7f958f: 2023-08-10T06:35:29,191 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-16916492264742,,1691649230428.a9e9ef626a68c685245826f3ca7f958f. 2023-08-10T06:35:29,192 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 2468e04f6179571f82a0adee7c1e6692, disabling compactions & flushes 2023-08-10T06:35:29,192 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1691649236001.2468e04f6179571f82a0adee7c1e6692. 2023-08-10T06:35:29,192 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1691649236001.2468e04f6179571f82a0adee7c1e6692. 2023-08-10T06:35:29,192 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1691649236001.2468e04f6179571f82a0adee7c1e6692. after waiting 0 ms 2023-08-10T06:35:29,192 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1691649236001.2468e04f6179571f82a0adee7c1e6692. 2023-08-10T06:35:29,195 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system_bulk/2468e04f6179571f82a0adee7c1e6692/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-08-10T06:35:29,196 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:35:29,196 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1691649236001.2468e04f6179571f82a0adee7c1e6692. 2023-08-10T06:35:29,196 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 2468e04f6179571f82a0adee7c1e6692: 2023-08-10T06:35:29,197 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1691649236001.2468e04f6179571f82a0adee7c1e6692. 2023-08-10T06:35:29,197 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 2a873d0a0a9423038307a20a4f0e1113, disabling compactions & flushes 2023-08-10T06:35:29,197 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1691649220813.2a873d0a0a9423038307a20a4f0e1113. 2023-08-10T06:35:29,197 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1691649220813.2a873d0a0a9423038307a20a4f0e1113. 2023-08-10T06:35:29,197 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1691649220813.2a873d0a0a9423038307a20a4f0e1113. after waiting 0 ms 2023-08-10T06:35:29,197 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1691649220813.2a873d0a0a9423038307a20a4f0e1113. 2023-08-10T06:35:29,198 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 2a873d0a0a9423038307a20a4f0e1113 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-08-10T06:35:29,216 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/hbase/meta/1588230740/.tmp/info/1d24562dc99946898bf2e2a4460ab115 is 159, key is ns2:test-16916492264741,,1691649228221.94f564ab80fb7e8519555fad2ae5463b./info:regioninfo/1691649229411/Put/seqid=0 2023-08-10T06:35:29,217 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/hbase/namespace/2a873d0a0a9423038307a20a4f0e1113/.tmp/info/3ae996412c8249b991124af333885dea is 45, key is default/info:d/1691649221802/Put/seqid=0 2023-08-10T06:35:29,386 DEBUG [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 2a873d0a0a9423038307a20a4f0e1113, 3e5d3d5b3b7bc4e6a979a8585e22b81f, 60ed3292def93d2ecd5862b25318ed47, 930ae4244d4cd65da422c67705de93c0, 94f564ab80fb7e8519555fad2ae5463b 2023-08-10T06:35:29,587 DEBUG [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 2a873d0a0a9423038307a20a4f0e1113, 3e5d3d5b3b7bc4e6a979a8585e22b81f, 60ed3292def93d2ecd5862b25318ed47, 930ae4244d4cd65da422c67705de93c0, 94f564ab80fb7e8519555fad2ae5463b 2023-08-10T06:35:29,625 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.55 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/hbase/meta/1588230740/.tmp/info/1d24562dc99946898bf2e2a4460ab115 2023-08-10T06:35:29,626 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/hbase/namespace/2a873d0a0a9423038307a20a4f0e1113/.tmp/info/3ae996412c8249b991124af333885dea 2023-08-10T06:35:29,631 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 1d24562dc99946898bf2e2a4460ab115 2023-08-10T06:35:29,631 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3ae996412c8249b991124af333885dea 2023-08-10T06:35:29,632 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/hbase/namespace/2a873d0a0a9423038307a20a4f0e1113/.tmp/info/3ae996412c8249b991124af333885dea as hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/hbase/namespace/2a873d0a0a9423038307a20a4f0e1113/info/3ae996412c8249b991124af333885dea 2023-08-10T06:35:29,637 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/hbase/namespace/2a873d0a0a9423038307a20a4f0e1113/info/3ae996412c8249b991124af333885dea, entries=7, sequenceid=11, filesize=5.1 K 2023-08-10T06:35:29,638 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 2a873d0a0a9423038307a20a4f0e1113 in 441ms, sequenceid=11, compaction requested=false 2023-08-10T06:35:29,641 DEBUG [StoreFileCloser-2a873d0a0a9423038307a20a4f0e1113-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3ae996412c8249b991124af333885dea 2023-08-10T06:35:29,645 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/hbase/namespace/2a873d0a0a9423038307a20a4f0e1113/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-08-10T06:35:29,646 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:35:29,647 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1691649220813.2a873d0a0a9423038307a20a4f0e1113. 2023-08-10T06:35:29,647 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 2a873d0a0a9423038307a20a4f0e1113: 2023-08-10T06:35:29,647 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1691649220813.2a873d0a0a9423038307a20a4f0e1113. 2023-08-10T06:35:29,648 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 930ae4244d4cd65da422c67705de93c0, disabling compactions & flushes 2023-08-10T06:35:29,648 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-16916492264743,,1691649232552.930ae4244d4cd65da422c67705de93c0. 2023-08-10T06:35:29,648 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16916492264743,,1691649232552.930ae4244d4cd65da422c67705de93c0. 2023-08-10T06:35:29,648 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16916492264743,,1691649232552.930ae4244d4cd65da422c67705de93c0. after waiting 0 ms 2023-08-10T06:35:29,648 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16916492264743,,1691649232552.930ae4244d4cd65da422c67705de93c0. 2023-08-10T06:35:29,651 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/hbase/meta/1588230740/.tmp/table/747c26689f8a48f28834d20c81802f04 is 59, key is ns2:test-16916492264741/table:state/1691649229421/Put/seqid=0 2023-08-10T06:35:29,652 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/ns4/test-16916492264743/930ae4244d4cd65da422c67705de93c0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-08-10T06:35:29,653 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:35:29,654 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-16916492264743,,1691649232552.930ae4244d4cd65da422c67705de93c0. 2023-08-10T06:35:29,654 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 930ae4244d4cd65da422c67705de93c0: 2023-08-10T06:35:29,654 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-16916492264743,,1691649232552.930ae4244d4cd65da422c67705de93c0. 2023-08-10T06:35:29,654 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 60ed3292def93d2ecd5862b25318ed47, disabling compactions & flushes 2023-08-10T06:35:29,654 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. 2023-08-10T06:35:29,654 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. 2023-08-10T06:35:29,654 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. after waiting 0 ms 2023-08-10T06:35:29,654 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. 2023-08-10T06:35:29,654 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 60ed3292def93d2ecd5862b25318ed47 2/2 column families, dataSize=1.94 KB heapSize=3.32 KB 2023-08-10T06:35:29,658 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/meta/5f51d59f99194024b709c04ff8ca4cd7 is 96, key is incrbackupset:hdfs://localhost:41489/backupUT/meta:ns2:test-16916492264741/1691649329105/Put/seqid=0 2023-08-10T06:35:29,757 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-08-10T06:35:29,757 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-08-10T06:35:29,787 DEBUG [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 3e5d3d5b3b7bc4e6a979a8585e22b81f, 60ed3292def93d2ecd5862b25318ed47, 94f564ab80fb7e8519555fad2ae5463b 2023-08-10T06:35:29,987 DEBUG [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 3e5d3d5b3b7bc4e6a979a8585e22b81f, 60ed3292def93d2ecd5862b25318ed47, 94f564ab80fb7e8519555fad2ae5463b 2023-08-10T06:35:30,056 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=714 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/hbase/meta/1588230740/.tmp/table/747c26689f8a48f28834d20c81802f04 2023-08-10T06:35:30,061 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 747c26689f8a48f28834d20c81802f04 2023-08-10T06:35:30,062 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=308 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/meta/5f51d59f99194024b709c04ff8ca4cd7 2023-08-10T06:35:30,062 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/hbase/meta/1588230740/.tmp/info/1d24562dc99946898bf2e2a4460ab115 as hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/hbase/meta/1588230740/info/1d24562dc99946898bf2e2a4460ab115 2023-08-10T06:35:30,067 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5f51d59f99194024b709c04ff8ca4cd7 2023-08-10T06:35:30,067 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 5f51d59f99194024b709c04ff8ca4cd7 2023-08-10T06:35:30,068 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/hbase/meta/1588230740/info/1d24562dc99946898bf2e2a4460ab115, entries=70, sequenceid=39, filesize=14.1 K 2023-08-10T06:35:30,069 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/hbase/meta/1588230740/.tmp/table/747c26689f8a48f28834d20c81802f04 as hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/hbase/meta/1588230740/table/747c26689f8a48f28834d20c81802f04 2023-08-10T06:35:30,070 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/session/6a8fa3fce82f403a97c78198a934c764 is 398, key is session:backup_1691649312626/session:context/1691649329086/Put/seqid=0 2023-08-10T06:35:30,074 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/hbase/meta/1588230740/table/747c26689f8a48f28834d20c81802f04, entries=14, sequenceid=39, filesize=5.8 K 2023-08-10T06:35:30,075 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~9.25 KB/9474, heapSize ~15.92 KB/16304, currentSize=0 B/0 for 1588230740 in 891ms, sequenceid=39, compaction requested=false 2023-08-10T06:35:30,078 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 1d24562dc99946898bf2e2a4460ab115 2023-08-10T06:35:30,080 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 747c26689f8a48f28834d20c81802f04 2023-08-10T06:35:30,084 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/hbase/meta/1588230740/recovered.edits/42.seqid, newMaxSeqId=42, maxSeqId=1 2023-08-10T06:35:30,084 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:35:30,085 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-08-10T06:35:30,085 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-08-10T06:35:30,085 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-08-10T06:35:30,085 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-08-10T06:35:30,188 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(1579): Waiting on 3 regions to close 2023-08-10T06:35:30,188 DEBUG [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(1583): Online Regions={60ed3292def93d2ecd5862b25318ed47=backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47., 3e5d3d5b3b7bc4e6a979a8585e22b81f=test-1691649226474,,1691649226962.3e5d3d5b3b7bc4e6a979a8585e22b81f., 94f564ab80fb7e8519555fad2ae5463b=ns2:test-16916492264741,,1691649228221.94f564ab80fb7e8519555fad2ae5463b.} 2023-08-10T06:35:30,188 DEBUG [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(1609): Waiting on 3e5d3d5b3b7bc4e6a979a8585e22b81f, 60ed3292def93d2ecd5862b25318ed47, 94f564ab80fb7e8519555fad2ae5463b 2023-08-10T06:35:30,388 DEBUG [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(1609): Waiting on 3e5d3d5b3b7bc4e6a979a8585e22b81f, 60ed3292def93d2ecd5862b25318ed47, 94f564ab80fb7e8519555fad2ae5463b 2023-08-10T06:35:30,474 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.64 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/session/6a8fa3fce82f403a97c78198a934c764 2023-08-10T06:35:30,480 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6a8fa3fce82f403a97c78198a934c764 2023-08-10T06:35:30,481 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6a8fa3fce82f403a97c78198a934c764 2023-08-10T06:35:30,481 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/meta/5f51d59f99194024b709c04ff8ca4cd7 as hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/5f51d59f99194024b709c04ff8ca4cd7 2023-08-10T06:35:30,486 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5f51d59f99194024b709c04ff8ca4cd7 2023-08-10T06:35:30,487 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/meta/5f51d59f99194024b709c04ff8ca4cd7, entries=4, sequenceid=97, filesize=5.4 K 2023-08-10T06:35:30,487 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/.tmp/session/6a8fa3fce82f403a97c78198a934c764 as hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/6a8fa3fce82f403a97c78198a934c764 2023-08-10T06:35:30,493 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6a8fa3fce82f403a97c78198a934c764 2023-08-10T06:35:30,493 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/session/6a8fa3fce82f403a97c78198a934c764, entries=5, sequenceid=97, filesize=5.7 K 2023-08-10T06:35:30,494 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~1.94 KB/1990, heapSize ~3.29 KB/3368, currentSize=0 B/0 for 60ed3292def93d2ecd5862b25318ed47 in 840ms, sequenceid=97, compaction requested=true 2023-08-10T06:35:30,496 DEBUG [StoreFileCloser-60ed3292def93d2ecd5862b25318ed47-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: ee5a7bc3977042ada997c293856052bb 2023-08-10T06:35:30,497 DEBUG [StoreFileCloser-60ed3292def93d2ecd5862b25318ed47-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: d4291c086d8c4791bd00119e10f93814 2023-08-10T06:35:30,499 DEBUG [StoreFileCloser-60ed3292def93d2ecd5862b25318ed47-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: b7540cf2901649cc88f83d61ac6d6a1f 2023-08-10T06:35:30,500 DEBUG [StoreFileCloser-60ed3292def93d2ecd5862b25318ed47-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 20be7fbcf544464eaf03f133b5735605 2023-08-10T06:35:30,502 DEBUG [StoreFileCloser-60ed3292def93d2ecd5862b25318ed47-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 10815475db68465796bf1f8cdb79c442 2023-08-10T06:35:30,503 DEBUG [StoreFileCloser-60ed3292def93d2ecd5862b25318ed47-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: d8d4cae980914e1ea84eb5038f273201 2023-08-10T06:35:30,505 DEBUG [StoreFileCloser-60ed3292def93d2ecd5862b25318ed47-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 055007204d8d4f4b8fccb41d0b6ece3c 2023-08-10T06:35:30,506 DEBUG [StoreFileCloser-60ed3292def93d2ecd5862b25318ed47-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 5f51d59f99194024b709c04ff8ca4cd7 2023-08-10T06:35:30,508 DEBUG [StoreFileCloser-60ed3292def93d2ecd5862b25318ed47-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6743ea07cad64eb59ec77e7feafa1733 2023-08-10T06:35:30,509 DEBUG [StoreFileCloser-60ed3292def93d2ecd5862b25318ed47-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 0eca886709dc4bc28ce13c5c7fbf8954 2023-08-10T06:35:30,511 DEBUG [StoreFileCloser-60ed3292def93d2ecd5862b25318ed47-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a9dfa7ceabf344a68958248aee65d7ab 2023-08-10T06:35:30,512 DEBUG [StoreFileCloser-60ed3292def93d2ecd5862b25318ed47-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: f9a52bd725e64f909af23b5393aacba4 2023-08-10T06:35:30,514 DEBUG [StoreFileCloser-60ed3292def93d2ecd5862b25318ed47-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 4b4b67835f074c859b336180b4bd2fe3 2023-08-10T06:35:30,515 DEBUG [StoreFileCloser-60ed3292def93d2ecd5862b25318ed47-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a642f64893574ae7b70f03df104c31d7 2023-08-10T06:35:30,517 DEBUG [StoreFileCloser-60ed3292def93d2ecd5862b25318ed47-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e87b306bd0fb4030ac6059a43fd7e96b 2023-08-10T06:35:30,518 DEBUG [StoreFileCloser-60ed3292def93d2ecd5862b25318ed47-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 90c628ec035c4038870ae2250993469d 2023-08-10T06:35:30,521 DEBUG [StoreFileCloser-60ed3292def93d2ecd5862b25318ed47-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6a8fa3fce82f403a97c78198a934c764 2023-08-10T06:35:30,527 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/backup/system/60ed3292def93d2ecd5862b25318ed47/recovered.edits/100.seqid, newMaxSeqId=100, maxSeqId=1 2023-08-10T06:35:30,528 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:35:30,528 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. 2023-08-10T06:35:30,528 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 60ed3292def93d2ecd5862b25318ed47: 2023-08-10T06:35:30,528 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1691649234874.60ed3292def93d2ecd5862b25318ed47. 2023-08-10T06:35:30,529 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 3e5d3d5b3b7bc4e6a979a8585e22b81f, disabling compactions & flushes 2023-08-10T06:35:30,529 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1691649226474,,1691649226962.3e5d3d5b3b7bc4e6a979a8585e22b81f. 2023-08-10T06:35:30,529 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1691649226474,,1691649226962.3e5d3d5b3b7bc4e6a979a8585e22b81f. 2023-08-10T06:35:30,529 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1691649226474,,1691649226962.3e5d3d5b3b7bc4e6a979a8585e22b81f. after waiting 0 ms 2023-08-10T06:35:30,529 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1691649226474,,1691649226962.3e5d3d5b3b7bc4e6a979a8585e22b81f. 2023-08-10T06:35:30,529 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 3e5d3d5b3b7bc4e6a979a8585e22b81f 1/1 column families, dataSize=7.13 KB heapSize=21.91 KB 2023-08-10T06:35:30,533 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/default/test-1691649226474/3e5d3d5b3b7bc4e6a979a8585e22b81f/.tmp/f/30c9315d4e954aecb817e027dfb7441b is 42, key is row-t1100/f:q1/1691649280098/Put/seqid=0 2023-08-10T06:35:30,588 DEBUG [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(1609): Waiting on 3e5d3d5b3b7bc4e6a979a8585e22b81f, 94f564ab80fb7e8519555fad2ae5463b 2023-08-10T06:35:30,788 DEBUG [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(1609): Waiting on 3e5d3d5b3b7bc4e6a979a8585e22b81f, 94f564ab80fb7e8519555fad2ae5463b 2023-08-10T06:35:30,938 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.13 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/default/test-1691649226474/3e5d3d5b3b7bc4e6a979a8585e22b81f/.tmp/f/30c9315d4e954aecb817e027dfb7441b 2023-08-10T06:35:30,945 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 30c9315d4e954aecb817e027dfb7441b 2023-08-10T06:35:30,945 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/default/test-1691649226474/3e5d3d5b3b7bc4e6a979a8585e22b81f/.tmp/f/30c9315d4e954aecb817e027dfb7441b as hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/default/test-1691649226474/3e5d3d5b3b7bc4e6a979a8585e22b81f/f/30c9315d4e954aecb817e027dfb7441b 2023-08-10T06:35:30,951 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/default/test-1691649226474/3e5d3d5b3b7bc4e6a979a8585e22b81f/f/30c9315d4e954aecb817e027dfb7441b, entries=198, sequenceid=304, filesize=12.6 K 2023-08-10T06:35:30,952 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~7.13 KB/7304, heapSize ~21.89 KB/22416, currentSize=0 B/0 for 3e5d3d5b3b7bc4e6a979a8585e22b81f in 423ms, sequenceid=304, compaction requested=false 2023-08-10T06:35:30,952 DEBUG [StoreFileCloser-3e5d3d5b3b7bc4e6a979a8585e22b81f-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2ff193057568456b8bd59abdcfa367df 2023-08-10T06:35:30,953 DEBUG [StoreFileCloser-3e5d3d5b3b7bc4e6a979a8585e22b81f-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 30c9315d4e954aecb817e027dfb7441b 2023-08-10T06:35:30,957 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/default/test-1691649226474/3e5d3d5b3b7bc4e6a979a8585e22b81f/recovered.edits/307.seqid, newMaxSeqId=307, maxSeqId=1 2023-08-10T06:35:30,958 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:35:30,958 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1691649226474,,1691649226962.3e5d3d5b3b7bc4e6a979a8585e22b81f. 2023-08-10T06:35:30,958 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 3e5d3d5b3b7bc4e6a979a8585e22b81f: 2023-08-10T06:35:30,958 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1691649226474,,1691649226962.3e5d3d5b3b7bc4e6a979a8585e22b81f. 2023-08-10T06:35:30,958 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 94f564ab80fb7e8519555fad2ae5463b, disabling compactions & flushes 2023-08-10T06:35:30,959 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-16916492264741,,1691649228221.94f564ab80fb7e8519555fad2ae5463b. 2023-08-10T06:35:30,959 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16916492264741,,1691649228221.94f564ab80fb7e8519555fad2ae5463b. 2023-08-10T06:35:30,959 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16916492264741,,1691649228221.94f564ab80fb7e8519555fad2ae5463b. after waiting 0 ms 2023-08-10T06:35:30,959 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16916492264741,,1691649228221.94f564ab80fb7e8519555fad2ae5463b. 2023-08-10T06:35:30,959 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 94f564ab80fb7e8519555fad2ae5463b 1/1 column families, dataSize=7.13 KB heapSize=21.91 KB 2023-08-10T06:35:30,962 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/ns2/test-16916492264741/94f564ab80fb7e8519555fad2ae5463b/.tmp/f/866fe7a26dcc4f788f6cdbedf00b4edc is 42, key is row-t1100/f:q1/1691649291648/Put/seqid=0 2023-08-10T06:35:30,989 DEBUG [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(1609): Waiting on 94f564ab80fb7e8519555fad2ae5463b 2023-08-10T06:35:31,189 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(1579): Waiting on 1 regions to close 2023-08-10T06:35:31,189 DEBUG [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(1583): Online Regions={94f564ab80fb7e8519555fad2ae5463b=ns2:test-16916492264741,,1691649228221.94f564ab80fb7e8519555fad2ae5463b.} 2023-08-10T06:35:31,189 DEBUG [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(1609): Waiting on 94f564ab80fb7e8519555fad2ae5463b 2023-08-10T06:35:31,368 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.13 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/ns2/test-16916492264741/94f564ab80fb7e8519555fad2ae5463b/.tmp/f/866fe7a26dcc4f788f6cdbedf00b4edc 2023-08-10T06:35:31,374 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 866fe7a26dcc4f788f6cdbedf00b4edc 2023-08-10T06:35:31,375 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/ns2/test-16916492264741/94f564ab80fb7e8519555fad2ae5463b/.tmp/f/866fe7a26dcc4f788f6cdbedf00b4edc as hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/ns2/test-16916492264741/94f564ab80fb7e8519555fad2ae5463b/f/866fe7a26dcc4f788f6cdbedf00b4edc 2023-08-10T06:35:31,380 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/ns2/test-16916492264741/94f564ab80fb7e8519555fad2ae5463b/f/866fe7a26dcc4f788f6cdbedf00b4edc, entries=198, sequenceid=304, filesize=12.6 K 2023-08-10T06:35:31,381 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~7.13 KB/7304, heapSize ~21.89 KB/22416, currentSize=0 B/0 for 94f564ab80fb7e8519555fad2ae5463b in 422ms, sequenceid=304, compaction requested=false 2023-08-10T06:35:31,381 DEBUG [StoreFileCloser-94f564ab80fb7e8519555fad2ae5463b-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: db978fb0aa084e4586c17125b4193ef8 2023-08-10T06:35:31,383 DEBUG [StoreFileCloser-94f564ab80fb7e8519555fad2ae5463b-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 866fe7a26dcc4f788f6cdbedf00b4edc 2023-08-10T06:35:31,389 DEBUG [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(1609): Waiting on 94f564ab80fb7e8519555fad2ae5463b 2023-08-10T06:35:31,390 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/data/ns2/test-16916492264741/94f564ab80fb7e8519555fad2ae5463b/recovered.edits/307.seqid, newMaxSeqId=307, maxSeqId=1 2023-08-10T06:35:31,390 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:35:31,391 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-16916492264741,,1691649228221.94f564ab80fb7e8519555fad2ae5463b. 2023-08-10T06:35:31,391 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 94f564ab80fb7e8519555fad2ae5463b: 2023-08-10T06:35:31,391 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-16916492264741,,1691649228221.94f564ab80fb7e8519555fad2ae5463b. 2023-08-10T06:35:31,589 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,44853,1691649217899; all regions closed. 2023-08-10T06:35:31,597 DEBUG [RS:0;jenkins-hbase3:44853 {}] wal.AbstractFSWAL(1062): Moved 2 WAL file(s) to /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs 2023-08-10T06:35:31,597 INFO [RS:0;jenkins-hbase3:44853 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C44853%2C1691649217899.meta:.meta(num 1691649316806) 2023-08-10T06:35:31,599 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/WALs/jenkins-hbase3.apache.org,44853,1691649217899/jenkins-hbase3.apache.org%2C44853%2C1691649217899.1691649316824 not finished, retry = 0 2023-08-10T06:35:31,708 DEBUG [RS:0;jenkins-hbase3:44853 {}] wal.AbstractFSWAL(1062): Moved 7 WAL file(s) to /user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/oldWALs 2023-08-10T06:35:31,708 INFO [RS:0;jenkins-hbase3:44853 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C44853%2C1691649217899:(num 1691649316824) 2023-08-10T06:35:31,708 DEBUG [RS:0;jenkins-hbase3:44853 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:35:31,708 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.LeaseManager(133): Closed leases 2023-08-10T06:35:31,709 INFO [RS:0;jenkins-hbase3:44853 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2023-08-10T06:35:31,709 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-08-10T06:35:31,711 INFO [RS:0;jenkins-hbase3:44853 {}] ipc.NettyRpcServer(213): Stopping server on /172.31.12.81:44853 2023-08-10T06:35:31,717 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-08-10T06:35:31,717 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rs/jenkins-hbase3.apache.org,44853,1691649217899 2023-08-10T06:35:31,719 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,44853,1691649217899] 2023-08-10T06:35:31,719 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,44853,1691649217899; numProcessing=1 2023-08-10T06:35:31,720 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(224): Node /1/draining/jenkins-hbase3.apache.org,44853,1691649217899 already deleted, retry=false 2023-08-10T06:35:31,720 INFO [RegionServerTracker-0 {}] master.ServerManager(569): Cluster shutdown set; jenkins-hbase3.apache.org,44853,1691649217899 expired; onlineServers=0 2023-08-10T06:35:31,720 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2513): ***** STOPPING region server 'jenkins-hbase3.apache.org,37947,1691649214739' ***** 2023-08-10T06:35:31,720 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2527): STOPPED: Cluster shutdown set; onlineServer=0 2023-08-10T06:35:31,721 DEBUG [M:0;jenkins-hbase3:37947 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52a96e8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-08-10T06:35:31,721 INFO [M:0;jenkins-hbase3:37947 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-08-10T06:35:31,728 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/master 2023-08-10T06:35:31,728 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:35:31,728 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-08-10T06:35:31,763 INFO [M:0;jenkins-hbase3:37947 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@77ee37e5{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-08-10T06:35:31,763 INFO [M:0;jenkins-hbase3:37947 {}] server.AbstractConnector(383): Stopped ServerConnector@432d25ff{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-08-10T06:35:31,763 INFO [M:0;jenkins-hbase3:37947 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-08-10T06:35:31,764 INFO [M:0;jenkins-hbase3:37947 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@53671fc7{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-08-10T06:35:31,764 INFO [M:0;jenkins-hbase3:37947 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@213b0a5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/hadoop.log.dir/,STOPPED} 2023-08-10T06:35:31,765 INFO [M:0;jenkins-hbase3:37947 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,37947,1691649214739 2023-08-10T06:35:31,765 INFO [M:0;jenkins-hbase3:37947 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,37947,1691649214739; all regions closed. 2023-08-10T06:35:31,765 DEBUG [M:0;jenkins-hbase3:37947 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:35:31,765 INFO [M:0;jenkins-hbase3:37947 {}] master.HMaster(1664): Stopping master jetty server 2023-08-10T06:35:31,765 INFO [M:0;jenkins-hbase3:37947 {}] server.AbstractConnector(383): Stopped ServerConnector@696643d9{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-08-10T06:35:31,766 DEBUG [M:0;jenkins-hbase3:37947 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-08-10T06:35:31,766 INFO [M:0;jenkins-hbase3:37947 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-08-10T06:35:31,766 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-08-10T06:35:31,766 DEBUG [M:0;jenkins-hbase3:37947 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x53d0f342 to 127.0.0.1:59679 2023-08-10T06:35:31,766 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1691649219478 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1691649219478,5,FailOnTimeoutGroup] 2023-08-10T06:35:31,766 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1691649219478 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1691649219478,5,FailOnTimeoutGroup] 2023-08-10T06:35:31,766 DEBUG [M:0;jenkins-hbase3:37947 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:35:31,766 DEBUG [M:0;jenkins-hbase3:37947 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-08-10T06:35:31,767 INFO [M:0;jenkins-hbase3:37947 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-08-10T06:35:31,767 DEBUG [M:0;jenkins-hbase3:37947 {}] master.HMaster(1687): Stopping service threads 2023-08-10T06:35:31,767 INFO [M:0;jenkins-hbase3:37947 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-08-10T06:35:31,767 ERROR [M:0;jenkins-hbase3:37947 {}] procedure2.ProcedureExecutor(655): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] 2023-08-10T06:35:31,767 INFO [M:0;jenkins-hbase3:37947 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-08-10T06:35:31,768 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-08-10T06:35:31,768 DEBUG [M:0;jenkins-hbase3:37947 {}] zookeeper.ZKUtil(347): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Unable to get data of znode /1/master because node does not exist (not an error) 2023-08-10T06:35:31,768 WARN [M:0;jenkins-hbase3:37947 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-08-10T06:35:31,768 INFO [M:0;jenkins-hbase3:37947 {}] assignment.AssignmentManager(382): Stopping assignment manager 2023-08-10T06:35:31,768 INFO [M:0;jenkins-hbase3:37947 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-08-10T06:35:31,769 DEBUG [M:0;jenkins-hbase3:37947 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-08-10T06:35:31,780 INFO [M:0;jenkins-hbase3:37947 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:35:31,780 DEBUG [M:0;jenkins-hbase3:37947 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:35:31,780 DEBUG [M:0;jenkins-hbase3:37947 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-08-10T06:35:31,781 DEBUG [M:0;jenkins-hbase3:37947 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:35:31,781 INFO [M:0;jenkins-hbase3:37947 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=201.98 KB heapSize=244.88 KB 2023-08-10T06:35:31,793 DEBUG [M:0;jenkins-hbase3:37947 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/632f1816caba4d0181c3325f24018566 is 88, key is hbase:meta,,1/info:sn/1691649220112/Put/seqid=0 2023-08-10T06:35:31,819 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-08-10T06:35:31,819 INFO [RS:0;jenkins-hbase3:44853 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,44853,1691649217899; zookeeper connection closed. 2023-08-10T06:35:31,819 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44853-0x10060efe5be0001, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-08-10T06:35:31,819 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@18418743 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@18418743 2023-08-10T06:35:31,819 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-08-10T06:35:32,197 INFO [M:0;jenkins-hbase3:37947 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/632f1816caba4d0181c3325f24018566 2023-08-10T06:35:32,202 DEBUG [M:0;jenkins-hbase3:37947 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 632f1816caba4d0181c3325f24018566 2023-08-10T06:35:32,215 DEBUG [M:0;jenkins-hbase3:37947 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2e74d4898654434a81b3c74e4a799069 is 986, key is \x00\x00\x00\x00\x00\x00\x00\x1D/proc:d/1691649237193/Put/seqid=0 2023-08-10T06:35:32,620 INFO [M:0;jenkins-hbase3:37947 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=201.39 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2e74d4898654434a81b3c74e4a799069 2023-08-10T06:35:32,625 INFO [M:0;jenkins-hbase3:37947 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2e74d4898654434a81b3c74e4a799069 2023-08-10T06:35:32,625 DEBUG [M:0;jenkins-hbase3:37947 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2e74d4898654434a81b3c74e4a799069 2023-08-10T06:35:32,637 DEBUG [M:0;jenkins-hbase3:37947 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4dc777b4c832449a9178ca902d22fa3c is 82, key is jenkins-hbase3.apache.org,44853,1691649217899/rs:state/1691649219588/Put/seqid=0 2023-08-10T06:35:33,042 INFO [M:0;jenkins-hbase3:37947 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4dc777b4c832449a9178ca902d22fa3c 2023-08-10T06:35:33,047 DEBUG [M:0;jenkins-hbase3:37947 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 4dc777b4c832449a9178ca902d22fa3c 2023-08-10T06:35:33,048 DEBUG [M:0;jenkins-hbase3:37947 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/632f1816caba4d0181c3325f24018566 as hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/632f1816caba4d0181c3325f24018566 2023-08-10T06:35:33,053 INFO [M:0;jenkins-hbase3:37947 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/632f1816caba4d0181c3325f24018566, entries=8, sequenceid=507, filesize=5.5 K 2023-08-10T06:35:33,054 DEBUG [M:0;jenkins-hbase3:37947 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2e74d4898654434a81b3c74e4a799069 as hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2e74d4898654434a81b3c74e4a799069 2023-08-10T06:35:33,058 INFO [M:0;jenkins-hbase3:37947 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2e74d4898654434a81b3c74e4a799069 2023-08-10T06:35:33,058 INFO [M:0;jenkins-hbase3:37947 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2e74d4898654434a81b3c74e4a799069, entries=53, sequenceid=507, filesize=16.6 K 2023-08-10T06:35:33,059 DEBUG [M:0;jenkins-hbase3:37947 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4dc777b4c832449a9178ca902d22fa3c as hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4dc777b4c832449a9178ca902d22fa3c 2023-08-10T06:35:33,063 INFO [M:0;jenkins-hbase3:37947 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:41489/user/jenkins/test-data/4163dfa4-73f7-f005-facf-a420f2d2158f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4dc777b4c832449a9178ca902d22fa3c, entries=1, sequenceid=507, filesize=5.1 K 2023-08-10T06:35:33,064 INFO [M:0;jenkins-hbase3:37947 {}] regionserver.HRegion(3022): Finished flush of dataSize ~201.98 KB/206829, heapSize ~244.58 KB/250448, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1283ms, sequenceid=507, compaction requested=false 2023-08-10T06:35:33,066 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 632f1816caba4d0181c3325f24018566 2023-08-10T06:35:33,067 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-proc-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2e74d4898654434a81b3c74e4a799069 2023-08-10T06:35:33,069 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-rs-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 4dc777b4c832449a9178ca902d22fa3c 2023-08-10T06:35:33,069 INFO [M:0;jenkins-hbase3:37947 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:35:33,069 DEBUG [M:0;jenkins-hbase3:37947 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-08-10T06:35:33,072 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-08-10T06:35:33,072 INFO [M:0;jenkins-hbase3:37947 {}] flush.MasterFlushTableProcedureManager(81): stop: server shutting down. 2023-08-10T06:35:33,072 INFO [M:0;jenkins-hbase3:37947 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-08-10T06:35:33,073 INFO [M:0;jenkins-hbase3:37947 {}] ipc.NettyRpcServer(213): Stopping server on /172.31.12.81:37947 2023-08-10T06:35:33,075 DEBUG [M:0;jenkins-hbase3:37947 {}] zookeeper.RecoverableZooKeeper(224): Node /1/rs/jenkins-hbase3.apache.org,37947,1691649214739 already deleted, retry=false 2023-08-10T06:35:33,177 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-08-10T06:35:33,177 INFO [M:0;jenkins-hbase3:37947 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,37947,1691649214739; zookeeper connection closed. 2023-08-10T06:35:33,177 DEBUG [Listener at localhost/46655-EventThread {}] zookeeper.ZKWatcher(604): master:37947-0x10060efe5be0000, quorum=127.0.0.1:59679, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-08-10T06:35:33,183 WARN [Listener at jenkins-hbase3.apache.org/36733 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-08-10T06:35:33,190 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-08-10T06:35:33,299 WARN [BP-548592812-172.31.12.81-1691649211357 heartbeating to localhost/127.0.0.1:41489 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-08-10T06:35:33,299 WARN [BP-548592812-172.31.12.81-1691649211357 heartbeating to localhost/127.0.0.1:41489 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-548592812-172.31.12.81-1691649211357 (Datanode Uuid 5cdc99a8-415e-4832-89c3-7ec2d3ac0fbf) service to localhost/127.0.0.1:41489 2023-08-10T06:35:33,301 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/cluster_727a6324-ee3a-516e-29f4-b0167fb1d918/dfs/data/data1/current/BP-548592812-172.31.12.81-1691649211357 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-08-10T06:35:33,301 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/7651a61c-d4d1-f8dc-ac5f-c5bc80be0d50/cluster_727a6324-ee3a-516e-29f4-b0167fb1d918/dfs/data/data2/current/BP-548592812-172.31.12.81-1691649211357 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-08-10T06:35:33,338 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-08-10T06:35:33,456 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2023-08-10T06:35:33,529 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-08-10T06:35:33,529 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-08-10T06:35:33,539 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-08-10T06:35:33,656 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-08-10T06:35:33,767 ERROR [Thread[Thread-275,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-08-10T06:35:33,767 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-08-10T06:35:33,874 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-08-10T06:35:33,880 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-08-10T06:35:33,882 ERROR [Thread[Thread-285,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-08-10T06:35:33,888 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-08-10T06:35:33,990 ERROR [Thread[Thread-250,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-08-10T06:35:33,990 INFO [Listener at jenkins-hbase3.apache.org/36733 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped