2023-07-22 12:36:45,950 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@13c9d689 2023-07-22 12:36:45,965 main DEBUG Took 0.012912 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-07-22 12:36:45,966 main DEBUG PluginManager 'Core' found 129 plugins 2023-07-22 12:36:45,966 main DEBUG PluginManager 'Level' found 0 plugins 2023-07-22 12:36:45,967 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-07-22 12:36:45,969 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:36:45,978 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-07-22 12:36:45,994 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:36:45,996 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:36:45,997 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:36:45,997 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:36:45,998 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:36:45,998 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:36:45,999 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:36:46,000 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:36:46,000 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:36:46,000 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:36:46,001 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:36:46,001 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:36:46,002 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:36:46,002 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:36:46,002 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:36:46,002 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:36:46,003 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:36:46,003 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:36:46,003 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:36:46,003 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:36:46,004 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:36:46,004 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:36:46,004 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:36:46,005 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:36:46,005 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:36:46,005 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-07-22 12:36:46,007 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:36:46,009 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-07-22 12:36:46,011 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-07-22 12:36:46,011 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-07-22 12:36:46,012 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-07-22 12:36:46,012 main DEBUG PluginManager 'Converter' found 47 plugins 2023-07-22 12:36:46,025 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-07-22 12:36:46,028 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-07-22 12:36:46,029 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-07-22 12:36:46,029 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-07-22 12:36:46,030 main DEBUG createAppenders(={Console}) 2023-07-22 12:36:46,031 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@13c9d689 initialized 2023-07-22 12:36:46,031 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@13c9d689 2023-07-22 12:36:46,031 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@13c9d689 OK. 2023-07-22 12:36:46,032 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-07-22 12:36:46,032 main DEBUG OutputStream closed 2023-07-22 12:36:46,032 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-07-22 12:36:46,032 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-07-22 12:36:46,033 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@15c43bd9 OK 2023-07-22 12:36:46,082 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-07-22 12:36:46,084 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-07-22 12:36:46,085 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-07-22 12:36:46,086 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-07-22 12:36:46,087 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-07-22 12:36:46,087 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-07-22 12:36:46,088 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-07-22 12:36:46,088 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-07-22 12:36:46,088 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-07-22 12:36:46,088 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-07-22 12:36:46,089 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-07-22 12:36:46,089 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-07-22 12:36:46,090 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-07-22 12:36:46,090 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-07-22 12:36:46,090 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-07-22 12:36:46,091 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-07-22 12:36:46,091 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-07-22 12:36:46,092 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-07-22 12:36:46,099 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-07-22 12:36:46,100 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@1b6e1eff) with optional ClassLoader: null 2023-07-22 12:36:46,100 main DEBUG Shutdown hook enabled. Registering a new one. 2023-07-22 12:36:46,101 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@1b6e1eff] started OK. 2023-07-22T12:36:46,145 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestBackupMultipleDeletes timeout: 13 mins 2023-07-22 12:36:46,149 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-07-22 12:36:46,150 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-07-22T12:36:46,466 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef 2023-07-22T12:36:46,471 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-07-22T12:36:46,473 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:36:46,474 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-07-22T12:36:46,478 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/cluster_a7b0bb94-85a8-a89c-f1a4-3863f52e4e30, deleteOnExit=true 2023-07-22T12:36:46,478 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-07-22T12:36:46,478 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/test.cache.data in system properties and HBase conf 2023-07-22T12:36:46,479 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/hadoop.tmp.dir in system properties and HBase conf 2023-07-22T12:36:46,479 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/hadoop.log.dir in system properties and HBase conf 2023-07-22T12:36:46,479 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/mapreduce.cluster.local.dir in system properties and HBase conf 2023-07-22T12:36:46,480 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-07-22T12:36:46,480 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-07-22T12:36:46,576 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-07-22T12:36:47,054 DEBUG [Time-limited test {}] fs.HFileSystem(308): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-07-22T12:36:47,059 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-07-22T12:36:47,059 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-07-22T12:36:47,060 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-07-22T12:36:47,060 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-07-22T12:36:47,060 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-07-22T12:36:47,061 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-07-22T12:36:47,061 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-07-22T12:36:47,061 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/dfs.journalnode.edits.dir in system properties and HBase conf 2023-07-22T12:36:47,061 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-07-22T12:36:47,062 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/nfs.dump.dir in system properties and HBase conf 2023-07-22T12:36:47,062 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/java.io.tmpdir in system properties and HBase conf 2023-07-22T12:36:47,062 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/dfs.journalnode.edits.dir in system properties and HBase conf 2023-07-22T12:36:47,063 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-07-22T12:36:47,063 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-07-22T12:36:47,577 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-07-22T12:36:47,582 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-07-22T12:36:47,914 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-07-22T12:36:48,097 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@3ebd6c2 via org.mortbay.log.Slf4jLog 2023-07-22T12:36:48,117 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:36:48,168 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:36:48,204 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/java.io.tmpdir/Jetty_localhost_39321_hdfs____e3qoty/webapp 2023-07-22T12:36:48,346 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:39321 2023-07-22T12:36:48,368 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-07-22T12:36:48,369 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-07-22T12:36:48,879 WARN [Listener at localhost/33313 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-07-22T12:36:48,969 WARN [Listener at localhost/33313 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-07-22T12:36:48,992 WARN [Listener at localhost/33313 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:36:48,998 INFO [Listener at localhost/33313 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:36:49,004 INFO [Listener at localhost/33313 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/java.io.tmpdir/Jetty_localhost_37463_datanode____.xc7q18/webapp 2023-07-22T12:36:49,123 INFO [Listener at localhost/33313 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:37463 2023-07-22T12:36:49,413 WARN [Listener at localhost/36365 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-07-22T12:36:49,703 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-07-22T12:36:49,886 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x91f157a477e58248: Processing first storage report for DS-bd231103-555e-4ded-91ba-364f1ff937fa from datanode fee69a25-9af2-43eb-8040-db30620d1a9d 2023-07-22T12:36:49,888 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x91f157a477e58248: from storage DS-bd231103-555e-4ded-91ba-364f1ff937fa node DatanodeRegistration(127.0.0.1:33835, datanodeUuid=fee69a25-9af2-43eb-8040-db30620d1a9d, infoPort=43451, infoSecurePort=0, ipcPort=36365, storageInfo=lv=-57;cid=testClusterID;nsid=256030056;c=1690029407680), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2023-07-22T12:36:49,888 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x91f157a477e58248: Processing first storage report for DS-d8985979-e379-43e1-a58b-e9e47f0d23e0 from datanode fee69a25-9af2-43eb-8040-db30620d1a9d 2023-07-22T12:36:49,888 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x91f157a477e58248: from storage DS-d8985979-e379-43e1-a58b-e9e47f0d23e0 node DatanodeRegistration(127.0.0.1:33835, datanodeUuid=fee69a25-9af2-43eb-8040-db30620d1a9d, infoPort=43451, infoSecurePort=0, ipcPort=36365, storageInfo=lv=-57;cid=testClusterID;nsid=256030056;c=1690029407680), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-07-22T12:36:49,934 DEBUG [Listener at localhost/36365 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef 2023-07-22T12:36:50,011 INFO [Listener at localhost/36365 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/cluster_a7b0bb94-85a8-a89c-f1a4-3863f52e4e30/zookeeper_0, clientPort=51445, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/cluster_a7b0bb94-85a8-a89c-f1a4-3863f52e4e30/zookeeper_0/version-2, dataDirSize=424 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/cluster_a7b0bb94-85a8-a89c-f1a4-3863f52e4e30/zookeeper_0/version-2, dataLogSize=424 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, serverId=0 2023-07-22T12:36:50,030 INFO [Listener at localhost/36365 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=51445 2023-07-22T12:36:50,037 INFO [Listener at localhost/36365 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:36:50,040 INFO [Listener at localhost/36365 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:36:50,705 INFO [Listener at localhost/36365 {}] util.FSUtils(461): Created version file at hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d with version=8 2023-07-22T12:36:50,705 INFO [Listener at localhost/36365 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/hbase-staging 2023-07-22T12:36:50,725 DEBUG [Listener at localhost/36365 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-07-22T12:36:50,725 DEBUG [Listener at localhost/36365 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-07-22T12:36:50,725 DEBUG [Listener at localhost/36365 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-07-22T12:36:50,725 DEBUG [Listener at localhost/36365 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-07-22T12:36:50,946 DEBUG [Listener at localhost/36365 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-07-22T12:36:51,099 INFO [Listener at localhost/36365 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-07-22T12:36:51,754 INFO [Listener at localhost/36365 {}] client.ConnectionUtils(127): master/jenkins-hbase3:0 server-side Connection retries=45 2023-07-22T12:36:51,801 INFO [Listener at localhost/36365 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-07-22T12:36:51,802 INFO [Listener at localhost/36365 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-07-22T12:36:51,802 INFO [Listener at localhost/36365 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-07-22T12:36:51,802 INFO [Listener at localhost/36365 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-07-22T12:36:51,802 INFO [Listener at localhost/36365 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-07-22T12:36:51,955 INFO [Listener at localhost/36365 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-07-22T12:36:52,062 DEBUG [Listener at localhost/36365 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-07-22T12:36:52,068 INFO [Listener at localhost/36365 {}] ipc.NettyRpcServer(177): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-07-22T12:36:52,119 DEBUG [Listener at localhost/36365 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 26912 (auto-detected) 2023-07-22T12:36:52,121 DEBUG [Listener at localhost/36365 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:1c:ff:fe:e2:13:ea (auto-detected) 2023-07-22T12:36:52,161 INFO [Listener at localhost/36365 {}] ipc.NettyRpcServer(144): Bind to /172.31.12.81:33617 2023-07-22T12:36:52,177 INFO [Listener at localhost/36365 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:36:52,180 INFO [Listener at localhost/36365 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:36:52,199 INFO [Listener at localhost/36365 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:33617 connecting to ZooKeeper ensemble=127.0.0.1:51445 2023-07-22T12:36:52,255 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:336170x0, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-07-22T12:36:52,259 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:33617-0x1000063a7240000 connected 2023-07-22T12:36:52,287 DEBUG [Listener at localhost/36365 {}] zookeeper.ZKUtil(113): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-07-22T12:36:52,291 DEBUG [Listener at localhost/36365 {}] zookeeper.ZKUtil(113): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-07-22T12:36:52,296 DEBUG [Listener at localhost/36365 {}] zookeeper.ZKUtil(113): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-07-22T12:36:52,306 DEBUG [Listener at localhost/36365 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33617 2023-07-22T12:36:52,307 DEBUG [Listener at localhost/36365 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33617 2023-07-22T12:36:52,307 DEBUG [Listener at localhost/36365 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33617 2023-07-22T12:36:52,308 DEBUG [Listener at localhost/36365 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33617 2023-07-22T12:36:52,308 DEBUG [Listener at localhost/36365 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33617 2023-07-22T12:36:52,363 INFO [Listener at localhost/36365 {}] log.Log(170): Logging initialized @7135ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-07-22T12:36:52,495 INFO [Listener at localhost/36365 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-07-22T12:36:52,496 INFO [Listener at localhost/36365 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-07-22T12:36:52,497 INFO [Listener at localhost/36365 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-07-22T12:36:52,499 INFO [Listener at localhost/36365 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-07-22T12:36:52,500 INFO [Listener at localhost/36365 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-07-22T12:36:52,500 INFO [Listener at localhost/36365 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-07-22T12:36:52,504 INFO [Listener at localhost/36365 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-07-22T12:36:52,648 INFO [Listener at localhost/36365 {}] http.HttpServer(1219): Jetty bound to port 36793 2023-07-22T12:36:52,651 INFO [Listener at localhost/36365 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-07-22T12:36:52,690 INFO [Listener at localhost/36365 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:36:52,695 INFO [Listener at localhost/36365 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@302eec96{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/hadoop.log.dir/,AVAILABLE} 2023-07-22T12:36:52,696 INFO [Listener at localhost/36365 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:36:52,696 INFO [Listener at localhost/36365 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@127c31ba{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-07-22T12:36:52,764 INFO [Listener at localhost/36365 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-07-22T12:36:52,778 INFO [Listener at localhost/36365 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-07-22T12:36:52,778 INFO [Listener at localhost/36365 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-07-22T12:36:52,780 INFO [Listener at localhost/36365 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-07-22T12:36:52,788 INFO [Listener at localhost/36365 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:36:54,006 INFO [Listener at localhost/36365 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@3552251f{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-07-22T12:36:54,017 INFO [Listener at localhost/36365 {}] server.AbstractConnector(333): Started ServerConnector@644ab3cd{HTTP/1.1, (http/1.1)}{0.0.0.0:36793} 2023-07-22T12:36:54,017 INFO [Listener at localhost/36365 {}] server.Server(415): Started @8790ms 2023-07-22T12:36:54,021 INFO [Listener at localhost/36365 {}] master.HMaster(484): hbase.rootdir=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d, hbase.cluster.distributed=false 2023-07-22T12:36:54,078 INFO [Listener at localhost/36365 {}] client.ConnectionUtils(127): regionserver/jenkins-hbase3:0 server-side Connection retries=45 2023-07-22T12:36:54,078 INFO [Listener at localhost/36365 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-07-22T12:36:54,078 INFO [Listener at localhost/36365 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-07-22T12:36:54,078 INFO [Listener at localhost/36365 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-07-22T12:36:54,079 INFO [Listener at localhost/36365 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-07-22T12:36:54,079 INFO [Listener at localhost/36365 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-07-22T12:36:54,087 INFO [Listener at localhost/36365 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-07-22T12:36:54,089 INFO [Listener at localhost/36365 {}] ipc.NettyRpcServer(177): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-07-22T12:36:54,090 INFO [Listener at localhost/36365 {}] ipc.NettyRpcServer(144): Bind to /172.31.12.81:39283 2023-07-22T12:36:54,093 INFO [Listener at localhost/36365 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-07-22T12:36:54,101 DEBUG [Listener at localhost/36365 {}] mob.MobFileCache(120): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-07-22T12:36:54,102 INFO [Listener at localhost/36365 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:36:54,104 INFO [Listener at localhost/36365 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:36:54,106 INFO [Listener at localhost/36365 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:39283 connecting to ZooKeeper ensemble=127.0.0.1:51445 2023-07-22T12:36:54,110 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:392830x0, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-07-22T12:36:54,111 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:39283-0x1000063a7240001 connected 2023-07-22T12:36:54,111 DEBUG [Listener at localhost/36365 {}] zookeeper.ZKUtil(113): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-07-22T12:36:54,112 DEBUG [Listener at localhost/36365 {}] zookeeper.ZKUtil(113): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-07-22T12:36:54,113 DEBUG [Listener at localhost/36365 {}] zookeeper.ZKUtil(113): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-07-22T12:36:54,114 DEBUG [Listener at localhost/36365 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39283 2023-07-22T12:36:54,114 DEBUG [Listener at localhost/36365 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39283 2023-07-22T12:36:54,114 DEBUG [Listener at localhost/36365 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39283 2023-07-22T12:36:54,118 DEBUG [Listener at localhost/36365 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39283 2023-07-22T12:36:54,118 DEBUG [Listener at localhost/36365 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39283 2023-07-22T12:36:54,121 INFO [Listener at localhost/36365 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-07-22T12:36:54,121 INFO [Listener at localhost/36365 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-07-22T12:36:54,122 INFO [Listener at localhost/36365 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-07-22T12:36:54,122 INFO [Listener at localhost/36365 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-07-22T12:36:54,122 INFO [Listener at localhost/36365 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-07-22T12:36:54,123 INFO [Listener at localhost/36365 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-07-22T12:36:54,123 INFO [Listener at localhost/36365 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-07-22T12:36:54,125 INFO [Listener at localhost/36365 {}] http.HttpServer(1219): Jetty bound to port 40513 2023-07-22T12:36:54,125 INFO [Listener at localhost/36365 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-07-22T12:36:54,128 INFO [Listener at localhost/36365 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:36:54,128 INFO [Listener at localhost/36365 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@44c954d0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/hadoop.log.dir/,AVAILABLE} 2023-07-22T12:36:54,129 INFO [Listener at localhost/36365 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:36:54,129 INFO [Listener at localhost/36365 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@f16de87{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-07-22T12:36:54,139 INFO [Listener at localhost/36365 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-07-22T12:36:54,140 INFO [Listener at localhost/36365 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-07-22T12:36:54,141 INFO [Listener at localhost/36365 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-07-22T12:36:54,141 INFO [Listener at localhost/36365 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-07-22T12:36:54,142 INFO [Listener at localhost/36365 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:36:54,146 INFO [Listener at localhost/36365 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@2234bbbc{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-07-22T12:36:54,147 INFO [Listener at localhost/36365 {}] server.AbstractConnector(333): Started ServerConnector@67109812{HTTP/1.1, (http/1.1)}{0.0.0.0:40513} 2023-07-22T12:36:54,148 INFO [Listener at localhost/36365 {}] server.Server(415): Started @8920ms 2023-07-22T12:36:54,153 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-07-22T12:36:54,158 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@5d154786{HTTP/1.1, (http/1.1)}{0.0.0.0:43151} 2023-07-22T12:36:54,159 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @8931ms 2023-07-22T12:36:54,159 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2387): Adding backup master ZNode /1/backup-masters/jenkins-hbase3.apache.org,33617,1690029410929 2023-07-22T12:36:54,172 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-07-22T12:36:54,172 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-07-22T12:36:54,176 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase3.apache.org,33617,1690029410929 2023-07-22T12:36:54,182 DEBUG [M:0;jenkins-hbase3:33617 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:33617 2023-07-22T12:36:54,196 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-07-22T12:36:54,196 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-07-22T12:36:54,196 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:36:54,196 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:36:54,197 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on existing znode=/1/master 2023-07-22T12:36:54,199 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase3.apache.org,33617,1690029410929 from backup master directory 2023-07-22T12:36:54,199 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on existing znode=/1/master 2023-07-22T12:36:54,202 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase3.apache.org,33617,1690029410929 2023-07-22T12:36:54,202 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-07-22T12:36:54,202 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-07-22T12:36:54,203 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-07-22T12:36:54,204 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,33617,1690029410929 2023-07-22T12:36:54,207 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-07-22T12:36:54,208 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-07-22T12:36:54,292 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(610): Created cluster ID file at hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/hbase.id with ID: 78e86040-b583-4951-b602-ef76433817b3 2023-07-22T12:36:54,340 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:36:54,359 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:36:54,359 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:36:54,418 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:36:54,420 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-07-22T12:36:54,450 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(270): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-07-22T12:36:54,450 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(276): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-07-22T12:36:54,452 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(285): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:283) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:310) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:950) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:573) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:570) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-07-22T12:36:54,466 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:950) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:573) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:570) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-07-22T12:36:54,468 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-07-22T12:36:54,550 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/MasterData/data/master/store 2023-07-22T12:36:54,980 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-07-22T12:36:54,981 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:36:54,981 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-07-22T12:36:54,982 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:36:54,982 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:36:54,982 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-07-22T12:36:54,982 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:36:54,982 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:36:54,982 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-07-22T12:36:54,984 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/MasterData/data/master/store/.initializing 2023-07-22T12:36:54,984 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/MasterData/WALs/jenkins-hbase3.apache.org,33617,1690029410929 2023-07-22T12:36:54,996 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-07-22T12:36:55,011 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C33617%2C1690029410929, suffix=, logDir=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/MasterData/WALs/jenkins-hbase3.apache.org,33617,1690029410929, archiveDir=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/MasterData/oldWALs, maxLogs=10 2023-07-22T12:36:55,040 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/MasterData/WALs/jenkins-hbase3.apache.org,33617,1690029410929/jenkins-hbase3.apache.org%2C33617%2C1690029410929.1690029415018, exclude list is [], retry=0 2023-07-22T12:36:55,066 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK] 2023-07-22T12:36:55,074 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:346) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:430) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:424) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:482) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:477) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:489) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-07-22T12:36:55,146 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/MasterData/WALs/jenkins-hbase3.apache.org,33617,1690029410929/jenkins-hbase3.apache.org%2C33617%2C1690029410929.1690029415018 2023-07-22T12:36:55,147 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK]] 2023-07-22T12:36:55,147 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:36:55,148 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:36:55,153 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:36:55,154 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:36:55,232 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:36:55,274 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-07-22T12:36:55,279 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:36:55,284 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:36:55,285 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:36:55,288 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-07-22T12:36:55,288 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:36:55,290 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:36:55,290 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:36:55,294 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-07-22T12:36:55,295 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:36:55,296 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:36:55,296 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:36:55,299 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-07-22T12:36:55,299 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:36:55,300 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:36:55,305 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:36:55,306 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:36:55,325 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-07-22T12:36:55,329 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:36:55,333 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:36:55,334 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9598899520, jitterRate=-0.10603281855583191}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-07-22T12:36:55,344 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-07-22T12:36:55,348 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-07-22T12:36:55,391 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d2c216c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:36:55,451 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(859): No meta location available on zookeeper, skip migrating... 2023-07-22T12:36:55,482 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-07-22T12:36:55,482 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(564): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-07-22T12:36:55,486 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-07-22T12:36:55,488 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(584): Recovered RegionProcedureStore lease in 1 msec 2023-07-22T12:36:55,495 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(598): Loaded RegionProcedureStore in 6 msec 2023-07-22T12:36:55,495 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-07-22T12:36:55,531 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-07-22T12:36:55,555 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-07-22T12:36:55,558 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-07-22T12:36:55,563 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(150): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-07-22T12:36:55,565 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-07-22T12:36:55,566 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-07-22T12:36:55,570 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-07-22T12:36:55,577 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-07-22T12:36:55,578 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-07-22T12:36:55,579 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-07-22T12:36:55,581 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-07-22T12:36:55,600 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-07-22T12:36:55,602 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-07-22T12:36:55,609 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-07-22T12:36:55,609 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-07-22T12:36:55,609 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:36:55,610 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:36:55,610 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(803): Active/primary master=jenkins-hbase3.apache.org,33617,1690029410929, sessionid=0x1000063a7240000, setting cluster-up flag (Was=false) 2023-07-22T12:36:55,621 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-07-22T12:36:55,632 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:36:55,632 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:36:55,638 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-07-22T12:36:55,639 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,33617,1690029410929 2023-07-22T12:36:55,644 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:36:55,644 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:36:55,651 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-07-22T12:36:55,652 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,33617,1690029410929 2023-07-22T12:36:55,658 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:36:55,658 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:36:55,664 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-07-22T12:36:55,666 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,33617,1690029410929 2023-07-22T12:36:55,762 DEBUG [RS:0;jenkins-hbase3:39283 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:39283 2023-07-22T12:36:55,764 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(997): ClusterId : 78e86040-b583-4951-b602-ef76433817b3 2023-07-22T12:36:55,766 INFO [RS:0;jenkins-hbase3:39283 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-07-22T12:36:55,768 DEBUG [RS:0;jenkins-hbase3:39283 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-07-22T12:36:55,775 DEBUG [RS:0;jenkins-hbase3:39283 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-07-22T12:36:55,775 DEBUG [RS:0;jenkins-hbase3:39283 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-07-22T12:36:55,778 DEBUG [RS:0;jenkins-hbase3:39283 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-07-22T12:36:55,778 DEBUG [RS:0;jenkins-hbase3:39283 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-07-22T12:36:55,781 DEBUG [RS:0;jenkins-hbase3:39283 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-07-22T12:36:55,782 DEBUG [RS:0;jenkins-hbase3:39283 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ac260c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:36:55,783 DEBUG [RS:0;jenkins-hbase3:39283 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e65b1cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-07-22T12:36:55,788 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-07-22T12:36:55,788 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-07-22T12:36:55,788 DEBUG [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-07-22T12:36:55,791 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(3023): reportForDuty to master=jenkins-hbase3.apache.org,33617,1690029410929 with isa=jenkins-hbase3.apache.org/172.31.12.81:39283, startcode=1690029414077 2023-07-22T12:36:55,812 DEBUG [RS:0;jenkins-hbase3:39283 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-07-22T12:36:55,846 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-07-22T12:36:55,859 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-07-22T12:36:55,867 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-07-22T12:36:55,878 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,33617,1690029410929 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-07-22T12:36:55,884 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-07-22T12:36:55,885 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-07-22T12:36:55,885 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-07-22T12:36:55,886 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-07-22T12:36:55,886 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-07-22T12:36:55,886 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:36:55,886 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-07-22T12:36:55,887 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:36:55,892 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1690029445892 2023-07-22T12:36:55,896 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-07-22T12:36:55,898 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56001, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-07-22T12:36:55,899 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-07-22T12:36:55,900 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-07-22T12:36:55,901 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-07-22T12:36:55,906 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:36:55,906 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-07-22T12:36:55,910 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33617 {}] ipc.MetricsHBaseServer(144): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3180) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:588) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-07-22T12:36:55,916 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-07-22T12:36:55,916 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-07-22T12:36:55,917 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-07-22T12:36:55,918 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-07-22T12:36:55,918 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-07-22T12:36:55,924 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-07-22T12:36:55,928 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-07-22T12:36:55,932 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-07-22T12:36:55,932 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-07-22T12:36:55,943 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1ebb090b to 127.0.0.1:51445 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:36:55,956 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@313ec673, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:36:55,957 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-07-22T12:36:55,960 DEBUG [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(3042): Master is not running yet 2023-07-22T12:36:55,960 WARN [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 100 ms and then retrying. 2023-07-22T12:36:55,961 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-07-22T12:36:55,962 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-07-22T12:36:55,964 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1690029415964,5,FailOnTimeoutGroup] 2023-07-22T12:36:55,964 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1690029415964,5,FailOnTimeoutGroup] 2023-07-22T12:36:55,965 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-07-22T12:36:55,965 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1634): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-07-22T12:36:55,967 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-07-22T12:36:55,967 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-07-22T12:36:56,062 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(3023): reportForDuty to master=jenkins-hbase3.apache.org,33617,1690029410929 with isa=jenkins-hbase3.apache.org/172.31.12.81:39283, startcode=1690029414077 2023-07-22T12:36:56,068 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33617 {}] master.ServerManager(403): Registering regionserver=jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:36:56,082 DEBUG [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d 2023-07-22T12:36:56,082 DEBUG [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:33313 2023-07-22T12:36:56,083 DEBUG [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=36793 2023-07-22T12:36:56,089 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-07-22T12:36:56,090 DEBUG [RS:0;jenkins-hbase3:39283 {}] zookeeper.ZKUtil(111): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:36:56,091 WARN [RS:0;jenkins-hbase3:39283 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-07-22T12:36:56,091 INFO [RS:0;jenkins-hbase3:39283 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-07-22T12:36:56,091 DEBUG [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:36:56,094 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,39283,1690029414077] 2023-07-22T12:36:56,110 DEBUG [RS:0;jenkins-hbase3:39283 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-07-22T12:36:56,120 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-07-22T12:36:56,138 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-07-22T12:36:56,144 INFO [RS:0;jenkins-hbase3:39283 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-07-22T12:36:56,145 INFO [RS:0;jenkins-hbase3:39283 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-07-22T12:36:56,146 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-07-22T12:36:56,156 INFO [RS:0;jenkins-hbase3:39283 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-07-22T12:36:56,156 DEBUG [RS:0;jenkins-hbase3:39283 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:36:56,156 DEBUG [RS:0;jenkins-hbase3:39283 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:36:56,156 DEBUG [RS:0;jenkins-hbase3:39283 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:36:56,156 DEBUG [RS:0;jenkins-hbase3:39283 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:36:56,156 DEBUG [RS:0;jenkins-hbase3:39283 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:36:56,157 DEBUG [RS:0;jenkins-hbase3:39283 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-07-22T12:36:56,157 DEBUG [RS:0;jenkins-hbase3:39283 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:36:56,157 DEBUG [RS:0;jenkins-hbase3:39283 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:36:56,157 DEBUG [RS:0;jenkins-hbase3:39283 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:36:56,157 DEBUG [RS:0;jenkins-hbase3:39283 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:36:56,157 DEBUG [RS:0;jenkins-hbase3:39283 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:36:56,157 DEBUG [RS:0;jenkins-hbase3:39283 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-07-22T12:36:56,158 INFO [RS:0;jenkins-hbase3:39283 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-07-22T12:36:56,159 INFO [RS:0;jenkins-hbase3:39283 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-07-22T12:36:56,159 INFO [RS:0;jenkins-hbase3:39283 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-07-22T12:36:56,159 INFO [RS:0;jenkins-hbase3:39283 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-07-22T12:36:56,159 INFO [RS:0;jenkins-hbase3:39283 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,39283,1690029414077-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-07-22T12:36:56,175 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-07-22T12:36:56,177 INFO [RS:0;jenkins-hbase3:39283 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,39283,1690029414077-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-07-22T12:36:56,194 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,39283,1690029414077 started 2023-07-22T12:36:56,194 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,39283,1690029414077, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:39283, sessionid=0x1000063a7240001 2023-07-22T12:36:56,195 DEBUG [RS:0;jenkins-hbase3:39283 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-07-22T12:36:56,195 DEBUG [RS:0;jenkins-hbase3:39283 {}] flush.RegionServerFlushTableProcedureManager(106): Start region server flush procedure manager jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:36:56,195 DEBUG [RS:0;jenkins-hbase3:39283 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,39283,1690029414077' 2023-07-22T12:36:56,195 DEBUG [RS:0;jenkins-hbase3:39283 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-07-22T12:36:56,195 DEBUG [RS:0;jenkins-hbase3:39283 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-07-22T12:36:56,196 DEBUG [RS:0;jenkins-hbase3:39283 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-07-22T12:36:56,196 DEBUG [RS:0;jenkins-hbase3:39283 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-07-22T12:36:56,196 DEBUG [RS:0;jenkins-hbase3:39283 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,39283,1690029414077' 2023-07-22T12:36:56,196 DEBUG [RS:0;jenkins-hbase3:39283 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:36:56,197 DEBUG [RS:0;jenkins-hbase3:39283 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:36:56,198 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-07-22T12:36:56,198 DEBUG [RS:0;jenkins-hbase3:39283 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-07-22T12:36:56,198 DEBUG [RS:0;jenkins-hbase3:39283 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-07-22T12:36:56,198 DEBUG [RS:0;jenkins-hbase3:39283 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:36:56,198 DEBUG [RS:0;jenkins-hbase3:39283 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,39283,1690029414077' 2023-07-22T12:36:56,198 DEBUG [RS:0;jenkins-hbase3:39283 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-07-22T12:36:56,199 DEBUG [RS:0;jenkins-hbase3:39283 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-07-22T12:36:56,199 DEBUG [RS:0;jenkins-hbase3:39283 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-07-22T12:36:56,200 INFO [RS:0;jenkins-hbase3:39283 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-07-22T12:36:56,200 INFO [RS:0;jenkins-hbase3:39283 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-07-22T12:36:56,310 INFO [RS:0;jenkins-hbase3:39283 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-07-22T12:36:56,313 INFO [RS:0;jenkins-hbase3:39283 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C39283%2C1690029414077, suffix=, logDir=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077, archiveDir=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs, maxLogs=32 2023-07-22T12:36:56,327 DEBUG [RS:0;jenkins-hbase3:39283 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029416315, exclude list is [], retry=0 2023-07-22T12:36:56,333 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK] 2023-07-22T12:36:56,337 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-07-22T12:36:56,338 INFO [RS:0;jenkins-hbase3:39283 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029416315 2023-07-22T12:36:56,338 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d 2023-07-22T12:36:56,338 DEBUG [RS:0;jenkins-hbase3:39283 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK]] 2023-07-22T12:36:56,753 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:36:56,755 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-07-22T12:36:56,757 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-07-22T12:36:56,757 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:36:56,758 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:36:56,758 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-07-22T12:36:56,760 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-07-22T12:36:56,760 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:36:56,761 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:36:56,761 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-07-22T12:36:56,763 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-07-22T12:36:56,763 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:36:56,764 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:36:56,765 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/hbase/meta/1588230740 2023-07-22T12:36:56,766 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/hbase/meta/1588230740 2023-07-22T12:36:56,768 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-07-22T12:36:56,770 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-07-22T12:36:56,774 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:36:56,775 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10322371840, jitterRate=-0.038654208183288574}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-07-22T12:36:56,776 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-07-22T12:36:56,776 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-07-22T12:36:56,776 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-07-22T12:36:56,777 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-07-22T12:36:56,777 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-07-22T12:36:56,777 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-07-22T12:36:56,778 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-07-22T12:36:56,778 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-07-22T12:36:56,782 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-07-22T12:36:56,782 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-07-22T12:36:56,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-07-22T12:36:56,809 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-07-22T12:36:56,813 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-07-22T12:36:56,965 DEBUG [jenkins-hbase3:33617 {}] assignment.AssignmentManager(2303): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-07-22T12:36:56,973 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:36:56,979 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,39283,1690029414077, state=OPENING 2023-07-22T12:36:56,989 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-07-22T12:36:56,995 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:36:56,995 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:36:56,996 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-07-22T12:36:56,996 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-07-22T12:36:56,998 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,39283,1690029414077}] 2023-07-22T12:36:57,189 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(721): New admin connection to jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:36:57,193 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=AdminService, sasl=false 2023-07-22T12:36:57,196 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:60984, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-07-22T12:36:57,208 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-07-22T12:36:57,232 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-07-22T12:36:57,233 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-07-22T12:36:57,236 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta, suffix=.meta, logDir=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077, archiveDir=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs, maxLogs=32 2023-07-22T12:36:57,248 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029417237.meta, exclude list is [], retry=0 2023-07-22T12:36:57,252 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK] 2023-07-22T12:36:57,256 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029417237.meta 2023-07-22T12:36:57,256 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK]] 2023-07-22T12:36:57,256 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:36:57,261 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:36:57,262 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-07-22T12:36:57,279 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-07-22T12:36:57,279 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-07-22T12:36:57,284 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-07-22T12:36:57,285 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:36:57,285 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-07-22T12:36:57,285 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-07-22T12:36:57,288 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-07-22T12:36:57,289 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-07-22T12:36:57,289 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:36:57,290 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:36:57,290 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-07-22T12:36:57,292 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-07-22T12:36:57,292 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:36:57,293 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:36:57,293 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-07-22T12:36:57,295 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-07-22T12:36:57,295 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:36:57,295 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:36:57,297 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/hbase/meta/1588230740 2023-07-22T12:36:57,299 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/hbase/meta/1588230740 2023-07-22T12:36:57,302 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-07-22T12:36:57,304 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-07-22T12:36:57,305 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11091143840, jitterRate=0.03294326364994049}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-07-22T12:36:57,306 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-07-22T12:36:57,318 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2552): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1690029417180 2023-07-22T12:36:57,338 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2579): Finished post open deploy task for hbase:meta,,1.1588230740 2023-07-22T12:36:57,338 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-07-22T12:36:57,340 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:36:57,341 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,39283,1690029414077, state=OPEN 2023-07-22T12:36:57,344 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-07-22T12:36:57,344 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-07-22T12:36:57,344 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-07-22T12:36:57,344 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-07-22T12:36:57,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=3, resume processing ppid=2 2023-07-22T12:36:57,349 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,39283,1690029414077 in 346 msec 2023-07-22T12:36:57,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=2, resume processing ppid=1 2023-07-22T12:36:57,359 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 555 msec 2023-07-22T12:36:57,364 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.6210 sec 2023-07-22T12:36:57,364 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1062): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1690029417364, completionTime=-1 2023-07-22T12:36:57,364 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(830): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-07-22T12:36:57,365 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1629): Joining cluster... 2023-07-22T12:36:57,574 DEBUG [hconnection-0xfc46a9a-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:36:57,577 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:60996, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:36:57,594 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1641): Number of RegionServers=1 2023-07-22T12:36:57,595 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1690029477595 2023-07-22T12:36:57,595 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1690029537595 2023-07-22T12:36:57,595 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1648): Joined the cluster in 230 msec 2023-07-22T12:36:57,618 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,33617,1690029410929-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-07-22T12:36:57,618 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,33617,1690029410929-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-07-22T12:36:57,618 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,33617,1690029410929-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-07-22T12:36:57,620 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:33617, period=300000, unit=MILLISECONDS is enabled. 2023-07-22T12:36:57,621 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-07-22T12:36:57,627 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-07-22T12:36:57,641 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-07-22T12:36:57,643 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2367): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-07-22T12:36:57,652 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-07-22T12:36:57,656 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:36:57,658 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:36:57,660 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:36:58,078 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => e1879fe3670023eba374a8cb2f79a944, NAME => 'hbase:namespace,,1690029417642.e1879fe3670023eba374a8cb2f79a944.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d 2023-07-22T12:36:58,490 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1690029417642.e1879fe3670023eba374a8cb2f79a944.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:36:58,490 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing e1879fe3670023eba374a8cb2f79a944, disabling compactions & flushes 2023-07-22T12:36:58,490 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1690029417642.e1879fe3670023eba374a8cb2f79a944. 2023-07-22T12:36:58,490 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1690029417642.e1879fe3670023eba374a8cb2f79a944. 2023-07-22T12:36:58,490 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1690029417642.e1879fe3670023eba374a8cb2f79a944. after waiting 0 ms 2023-07-22T12:36:58,490 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1690029417642.e1879fe3670023eba374a8cb2f79a944. 2023-07-22T12:36:58,490 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1690029417642.e1879fe3670023eba374a8cb2f79a944. 2023-07-22T12:36:58,490 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for e1879fe3670023eba374a8cb2f79a944: 2023-07-22T12:36:58,492 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:36:58,506 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1690029417642.e1879fe3670023eba374a8cb2f79a944.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1690029418494"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029418494"}]},"ts":"1690029418494"} 2023-07-22T12:36:58,541 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:36:58,543 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:36:58,548 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029418543"}]},"ts":"1690029418543"} 2023-07-22T12:36:58,552 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-07-22T12:36:58,557 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=e1879fe3670023eba374a8cb2f79a944, ASSIGN}] 2023-07-22T12:36:58,559 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=e1879fe3670023eba374a8cb2f79a944, ASSIGN 2023-07-22T12:36:58,561 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=e1879fe3670023eba374a8cb2f79a944, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,39283,1690029414077; forceNewPlan=false, retain=false 2023-07-22T12:36:58,712 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=e1879fe3670023eba374a8cb2f79a944, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:36:58,716 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure e1879fe3670023eba374a8cb2f79a944, server=jenkins-hbase3.apache.org,39283,1690029414077}] 2023-07-22T12:36:58,875 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1690029417642.e1879fe3670023eba374a8cb2f79a944. 2023-07-22T12:36:58,876 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => e1879fe3670023eba374a8cb2f79a944, NAME => 'hbase:namespace,,1690029417642.e1879fe3670023eba374a8cb2f79a944.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:36:58,877 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:36:58,877 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace e1879fe3670023eba374a8cb2f79a944 2023-07-22T12:36:58,877 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1690029417642.e1879fe3670023eba374a8cb2f79a944.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:36:58,877 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for e1879fe3670023eba374a8cb2f79a944 2023-07-22T12:36:58,877 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for e1879fe3670023eba374a8cb2f79a944 2023-07-22T12:36:58,880 INFO [StoreOpener-e1879fe3670023eba374a8cb2f79a944-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region e1879fe3670023eba374a8cb2f79a944 2023-07-22T12:36:58,882 INFO [StoreOpener-e1879fe3670023eba374a8cb2f79a944-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1879fe3670023eba374a8cb2f79a944 columnFamilyName info 2023-07-22T12:36:58,882 DEBUG [StoreOpener-e1879fe3670023eba374a8cb2f79a944-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:36:58,883 INFO [StoreOpener-e1879fe3670023eba374a8cb2f79a944-1 {}] regionserver.HStore(324): Store=e1879fe3670023eba374a8cb2f79a944/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:36:58,885 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/hbase/namespace/e1879fe3670023eba374a8cb2f79a944 2023-07-22T12:36:58,885 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/hbase/namespace/e1879fe3670023eba374a8cb2f79a944 2023-07-22T12:36:58,889 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for e1879fe3670023eba374a8cb2f79a944 2023-07-22T12:36:58,893 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/hbase/namespace/e1879fe3670023eba374a8cb2f79a944/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:36:58,894 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened e1879fe3670023eba374a8cb2f79a944; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11592992320, jitterRate=0.07968154549598694}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:36:58,894 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for e1879fe3670023eba374a8cb2f79a944: 2023-07-22T12:36:58,896 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2552): Post open deploy tasks for hbase:namespace,,1690029417642.e1879fe3670023eba374a8cb2f79a944., pid=6, masterSystemTime=1690029418869 2023-07-22T12:36:58,900 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2579): Finished post open deploy task for hbase:namespace,,1690029417642.e1879fe3670023eba374a8cb2f79a944. 2023-07-22T12:36:58,900 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1690029417642.e1879fe3670023eba374a8cb2f79a944. 2023-07-22T12:36:58,901 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=e1879fe3670023eba374a8cb2f79a944, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:36:58,907 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=6, resume processing ppid=5 2023-07-22T12:36:58,908 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure e1879fe3670023eba374a8cb2f79a944, server=jenkins-hbase3.apache.org,39283,1690029414077 in 188 msec 2023-07-22T12:36:58,911 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=5, resume processing ppid=4 2023-07-22T12:36:58,911 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=e1879fe3670023eba374a8cb2f79a944, ASSIGN in 351 msec 2023-07-22T12:36:58,912 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:36:58,912 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029418912"}]},"ts":"1690029418912"} 2023-07-22T12:36:58,915 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-07-22T12:36:58,918 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:36:58,920 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 1.2740 sec 2023-07-22T12:36:58,957 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-07-22T12:36:58,959 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-07-22T12:36:58,959 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:36:58,959 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:36:59,008 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-07-22T12:36:59,025 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:36:59,030 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 29 msec 2023-07-22T12:36:59,032 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-07-22T12:36:59,042 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:36:59,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 14 msec 2023-07-22T12:36:59,059 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-07-22T12:36:59,061 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-07-22T12:36:59,062 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1192): Master has completed initialization 4.858sec 2023-07-22T12:36:59,064 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-07-22T12:36:59,067 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-07-22T12:36:59,068 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-07-22T12:36:59,069 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-07-22T12:36:59,069 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-07-22T12:36:59,072 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,33617,1690029410929-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-07-22T12:36:59,073 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,33617,1690029410929-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-07-22T12:36:59,090 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1295): Balancer post startup initialization complete, took 0 seconds 2023-07-22T12:36:59,092 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-07-22T12:36:59,120 DEBUG [Listener at localhost/36365 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x43801001 to 127.0.0.1:51445 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:36:59,125 DEBUG [Listener at localhost/36365 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52584ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:36:59,130 DEBUG [Listener at localhost/36365 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-07-22T12:36:59,130 DEBUG [Listener at localhost/36365 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-07-22T12:36:59,144 DEBUG [hconnection-0x4857f49e-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:36:59,156 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:60998, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:36:59,166 INFO [Listener at localhost/36365 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,33617,1690029410929 2023-07-22T12:36:59,166 INFO [Listener at localhost/36365 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-07-22T12:36:59,166 INFO [Listener at localhost/36365 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/test.cache.data in system properties and HBase conf 2023-07-22T12:36:59,166 INFO [Listener at localhost/36365 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/hadoop.tmp.dir in system properties and HBase conf 2023-07-22T12:36:59,166 INFO [Listener at localhost/36365 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/hadoop.log.dir in system properties and HBase conf 2023-07-22T12:36:59,166 INFO [Listener at localhost/36365 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/mapreduce.cluster.local.dir in system properties and HBase conf 2023-07-22T12:36:59,166 INFO [Listener at localhost/36365 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-07-22T12:36:59,166 INFO [Listener at localhost/36365 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-07-22T12:36:59,166 INFO [Listener at localhost/36365 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-07-22T12:36:59,167 INFO [Listener at localhost/36365 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-07-22T12:36:59,167 INFO [Listener at localhost/36365 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-07-22T12:36:59,167 INFO [Listener at localhost/36365 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-07-22T12:36:59,167 INFO [Listener at localhost/36365 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-07-22T12:36:59,167 INFO [Listener at localhost/36365 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-07-22T12:36:59,167 INFO [Listener at localhost/36365 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-07-22T12:36:59,167 INFO [Listener at localhost/36365 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/dfs.journalnode.edits.dir in system properties and HBase conf 2023-07-22T12:36:59,167 INFO [Listener at localhost/36365 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-07-22T12:36:59,167 INFO [Listener at localhost/36365 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/nfs.dump.dir in system properties and HBase conf 2023-07-22T12:36:59,167 INFO [Listener at localhost/36365 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/java.io.tmpdir in system properties and HBase conf 2023-07-22T12:36:59,167 INFO [Listener at localhost/36365 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/dfs.journalnode.edits.dir in system properties and HBase conf 2023-07-22T12:36:59,167 INFO [Listener at localhost/36365 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-07-22T12:36:59,167 INFO [Listener at localhost/36365 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-07-22T12:37:00,580 WARN [Thread-249 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:37:00,955 INFO [Thread-249 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:37:00,960 INFO [Thread-249 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_46683_jobhistory____.ct55vr/webapp 2023-07-22T12:37:01,095 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-07-22T12:37:01,095 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-07-22T12:37:01,101 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-07-22T12:37:01,101 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-07-22T12:37:01,103 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:37:01,103 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-07-22T12:37:01,103 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-07-22T12:37:01,103 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-07-22T12:37:01,103 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-07-22T12:37:01,103 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-07-22T12:37:01,134 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-07-22T12:37:01,135 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-07-22T12:37:01,135 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-07-22T12:37:01,138 INFO [Thread-249 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-07-22T12:37:01,227 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:37:01,351 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:37:01,630 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-07-22T12:37:01,661 INFO [Thread-249 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:46683 2023-07-22T12:37:02,074 WARN [Listener at jenkins-hbase3.apache.org/44461 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:37:02,117 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-07-22T12:37:02,119 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-07-22T12:37:02,160 WARN [Listener at jenkins-hbase3.apache.org/44461 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-07-22T12:37:02,161 INFO [Listener at jenkins-hbase3.apache.org/44461 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:37:02,167 INFO [Listener at jenkins-hbase3.apache.org/44461 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_39273_cluster____1v8drv/webapp 2023-07-22T12:37:02,235 INFO [Listener at jenkins-hbase3.apache.org/44461 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-07-22T12:37:02,235 INFO [Listener at jenkins-hbase3.apache.org/44461 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-07-22T12:37:02,236 INFO [Listener at jenkins-hbase3.apache.org/44461 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-07-22T12:37:02,236 INFO [Listener at jenkins-hbase3.apache.org/44461 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-07-22T12:37:02,243 INFO [Listener at jenkins-hbase3.apache.org/44461 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:37:02,447 INFO [Listener at jenkins-hbase3.apache.org/44461 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:37:02,531 INFO [Listener at jenkins-hbase3.apache.org/44461 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:37:02,537 INFO [Listener at jenkins-hbase3.apache.org/44461 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:39273 2023-07-22T12:37:02,831 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:37:02,904 WARN [Listener at jenkins-hbase3.apache.org/43659 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:37:02,931 WARN [Listener at jenkins-hbase3.apache.org/43659 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-07-22T12:37:02,932 INFO [Listener at jenkins-hbase3.apache.org/43659 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:37:02,945 INFO [Listener at jenkins-hbase3.apache.org/43659 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_36985_node____.nrwqi6/webapp 2023-07-22T12:37:03,005 INFO [Listener at jenkins-hbase3.apache.org/43659 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-07-22T12:37:03,005 INFO [Listener at jenkins-hbase3.apache.org/43659 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-07-22T12:37:03,005 INFO [Listener at jenkins-hbase3.apache.org/43659 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-07-22T12:37:03,005 INFO [Listener at jenkins-hbase3.apache.org/43659 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-07-22T12:37:03,014 INFO [Listener at jenkins-hbase3.apache.org/43659 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:37:03,036 INFO [Listener at jenkins-hbase3.apache.org/43659 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:37:03,123 INFO [Listener at jenkins-hbase3.apache.org/43659 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:37:03,133 INFO [Listener at jenkins-hbase3.apache.org/43659 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:36985 2023-07-22T12:37:03,283 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:37:03,289 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-07-22T12:37:03,289 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:37:03,295 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_32941_node____t80edy/webapp 2023-07-22T12:37:03,346 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-07-22T12:37:03,346 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-07-22T12:37:03,346 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-07-22T12:37:03,346 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-07-22T12:37:03,357 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:37:03,382 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:37:03,475 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:37:03,486 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:32941 2023-07-22T12:37:03,533 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-07-22T12:37:03,533 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:33313/backupUT 2023-07-22T12:37:03,548 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:37:03,552 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:44526, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:37:03,562 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns1'} 2023-07-22T12:37:03,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure2.ProcedureExecutor(1033): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-07-22T12:37:03,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=9 2023-07-22T12:37:03,597 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:37:03,607 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 42 msec 2023-07-22T12:37:03,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=9 2023-07-22T12:37:03,689 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns2'} 2023-07-22T12:37:03,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure2.ProcedureExecutor(1033): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-07-22T12:37:03,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=10 2023-07-22T12:37:03,704 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:37:03,711 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 20 msec 2023-07-22T12:37:03,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=10 2023-07-22T12:37:03,801 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns3'} 2023-07-22T12:37:03,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure2.ProcedureExecutor(1033): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-07-22T12:37:03,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=11 2023-07-22T12:37:03,815 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:37:03,821 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 18 msec 2023-07-22T12:37:03,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=11 2023-07-22T12:37:03,912 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns4'} 2023-07-22T12:37:03,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure2.ProcedureExecutor(1033): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-07-22T12:37:03,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=12 2023-07-22T12:37:03,926 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:37:03,931 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 17 msec 2023-07-22T12:37:04,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=12 2023-07-22T12:37:04,034 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'test-1690029423533', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:37:04,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure2.ProcedureExecutor(1033): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1690029423533 2023-07-22T12:37:04,039 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1690029423533 execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:37:04,040 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:37:04,041 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1690029423533 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:37:04,044 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "test-1690029423533" procId is: 13 2023-07-22T12:37:04,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:37:04,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:37:04,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:37:04,474 INFO [RegionOpenAndInit-test-1690029423533-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => dce7657497bf6b1da7900abd4083b1b8, NAME => 'test-1690029423533,,1690029424029.dce7657497bf6b1da7900abd4083b1b8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1690029423533', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d 2023-07-22T12:37:04,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:37:04,891 DEBUG [RegionOpenAndInit-test-1690029423533-pool-0 {}] regionserver.HRegion(888): Instantiated test-1690029423533,,1690029424029.dce7657497bf6b1da7900abd4083b1b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:37:04,891 DEBUG [RegionOpenAndInit-test-1690029423533-pool-0 {}] regionserver.HRegion(1668): Closing dce7657497bf6b1da7900abd4083b1b8, disabling compactions & flushes 2023-07-22T12:37:04,891 INFO [RegionOpenAndInit-test-1690029423533-pool-0 {}] regionserver.HRegion(1690): Closing region test-1690029423533,,1690029424029.dce7657497bf6b1da7900abd4083b1b8. 2023-07-22T12:37:04,891 DEBUG [RegionOpenAndInit-test-1690029423533-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1690029423533,,1690029424029.dce7657497bf6b1da7900abd4083b1b8. 2023-07-22T12:37:04,891 DEBUG [RegionOpenAndInit-test-1690029423533-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1690029423533,,1690029424029.dce7657497bf6b1da7900abd4083b1b8. after waiting 0 ms 2023-07-22T12:37:04,891 DEBUG [RegionOpenAndInit-test-1690029423533-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1690029423533,,1690029424029.dce7657497bf6b1da7900abd4083b1b8. 2023-07-22T12:37:04,892 INFO [RegionOpenAndInit-test-1690029423533-pool-0 {}] regionserver.HRegion(1909): Closed test-1690029423533,,1690029424029.dce7657497bf6b1da7900abd4083b1b8. 2023-07-22T12:37:04,892 DEBUG [RegionOpenAndInit-test-1690029423533-pool-0 {}] regionserver.HRegion(1622): Region close journal for dce7657497bf6b1da7900abd4083b1b8: 2023-07-22T12:37:04,893 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1690029423533 execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:37:04,894 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1690029423533,,1690029424029.dce7657497bf6b1da7900abd4083b1b8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1690029424893"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029424893"}]},"ts":"1690029424893"} 2023-07-22T12:37:04,896 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:37:04,898 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1690029423533 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:37:04,898 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1690029423533","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029424898"}]},"ts":"1690029424898"} 2023-07-22T12:37:04,900 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1690029423533, state=ENABLING in hbase:meta 2023-07-22T12:37:04,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1690029423533, region=dce7657497bf6b1da7900abd4083b1b8, ASSIGN}] 2023-07-22T12:37:04,907 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1690029423533, region=dce7657497bf6b1da7900abd4083b1b8, ASSIGN 2023-07-22T12:37:04,909 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1690029423533, region=dce7657497bf6b1da7900abd4083b1b8, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,39283,1690029414077; forceNewPlan=false, retain=false 2023-07-22T12:37:05,060 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=dce7657497bf6b1da7900abd4083b1b8, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:05,064 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure dce7657497bf6b1da7900abd4083b1b8, server=jenkins-hbase3.apache.org,39283,1690029414077}] 2023-07-22T12:37:05,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:37:05,223 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1690029423533,,1690029424029.dce7657497bf6b1da7900abd4083b1b8. 2023-07-22T12:37:05,223 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => dce7657497bf6b1da7900abd4083b1b8, NAME => 'test-1690029423533,,1690029424029.dce7657497bf6b1da7900abd4083b1b8.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:37:05,224 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:37:05,224 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1690029423533 dce7657497bf6b1da7900abd4083b1b8 2023-07-22T12:37:05,224 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1690029423533,,1690029424029.dce7657497bf6b1da7900abd4083b1b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:37:05,224 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for dce7657497bf6b1da7900abd4083b1b8 2023-07-22T12:37:05,224 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for dce7657497bf6b1da7900abd4083b1b8 2023-07-22T12:37:05,227 INFO [StoreOpener-dce7657497bf6b1da7900abd4083b1b8-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region dce7657497bf6b1da7900abd4083b1b8 2023-07-22T12:37:05,229 INFO [StoreOpener-dce7657497bf6b1da7900abd4083b1b8-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dce7657497bf6b1da7900abd4083b1b8 columnFamilyName f 2023-07-22T12:37:05,229 DEBUG [StoreOpener-dce7657497bf6b1da7900abd4083b1b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:37:05,230 INFO [StoreOpener-dce7657497bf6b1da7900abd4083b1b8-1 {}] regionserver.HStore(324): Store=dce7657497bf6b1da7900abd4083b1b8/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:37:05,231 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/default/test-1690029423533/dce7657497bf6b1da7900abd4083b1b8 2023-07-22T12:37:05,232 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/default/test-1690029423533/dce7657497bf6b1da7900abd4083b1b8 2023-07-22T12:37:05,237 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for dce7657497bf6b1da7900abd4083b1b8 2023-07-22T12:37:05,245 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/default/test-1690029423533/dce7657497bf6b1da7900abd4083b1b8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:37:05,246 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened dce7657497bf6b1da7900abd4083b1b8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10769043360, jitterRate=0.002945318818092346}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:37:05,247 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for dce7657497bf6b1da7900abd4083b1b8: 2023-07-22T12:37:05,248 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2552): Post open deploy tasks for test-1690029423533,,1690029424029.dce7657497bf6b1da7900abd4083b1b8., pid=15, masterSystemTime=1690029425216 2023-07-22T12:37:05,251 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2579): Finished post open deploy task for test-1690029423533,,1690029424029.dce7657497bf6b1da7900abd4083b1b8. 2023-07-22T12:37:05,251 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1690029423533,,1690029424029.dce7657497bf6b1da7900abd4083b1b8. 2023-07-22T12:37:05,253 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=dce7657497bf6b1da7900abd4083b1b8, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:05,259 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=15, resume processing ppid=14 2023-07-22T12:37:05,259 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure dce7657497bf6b1da7900abd4083b1b8, server=jenkins-hbase3.apache.org,39283,1690029414077 in 193 msec 2023-07-22T12:37:05,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=14, resume processing ppid=13 2023-07-22T12:37:05,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1690029423533, region=dce7657497bf6b1da7900abd4083b1b8, ASSIGN in 354 msec 2023-07-22T12:37:05,264 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1690029423533 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:37:05,264 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1690029423533","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029425264"}]},"ts":"1690029425264"} 2023-07-22T12:37:05,266 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1690029423533, state=ENABLED in hbase:meta 2023-07-22T12:37:05,269 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1690029423533 execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:37:05,271 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1690029423533 in 1.2340 sec 2023-07-22T12:37:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:37:06,162 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: default:test-1690029423533, procId: 13 completed 2023-07-22T12:37:06,162 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x11973d96 to 127.0.0.1:51445 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:37:06,168 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6693fb1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:37:06,170 DEBUG [hconnection-0x5105cd1d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:37:06,173 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:58882, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:37:06,178 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39283 {}] regionserver.HRegion(8210): writing data to region test-1690029423533,,1690029424029.dce7657497bf6b1da7900abd4083b1b8. with WAL disabled. Data may be lost in the event of a crash. 2023-07-22T12:37:06,287 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns2:test-16900294235331', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:37:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure2.ProcedureExecutor(1033): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-16900294235331 2023-07-22T12:37:06,292 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16900294235331 execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:37:06,292 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:37:06,292 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "test-16900294235331" procId is: 16 2023-07-22T12:37:06,293 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-16900294235331 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:37:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-07-22T12:37:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-07-22T12:37:06,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-07-22T12:37:06,713 INFO [RegionOpenAndInit-ns2:test-16900294235331-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 626922efc5eebf51b8bc065f9cae11de, NAME => 'ns2:test-16900294235331,,1690029426287.626922efc5eebf51b8bc065f9cae11de.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16900294235331', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d 2023-07-22T12:37:06,726 DEBUG [RegionOpenAndInit-ns2:test-16900294235331-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16900294235331,,1690029426287.626922efc5eebf51b8bc065f9cae11de.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:37:06,726 DEBUG [RegionOpenAndInit-ns2:test-16900294235331-pool-0 {}] regionserver.HRegion(1668): Closing 626922efc5eebf51b8bc065f9cae11de, disabling compactions & flushes 2023-07-22T12:37:06,726 INFO [RegionOpenAndInit-ns2:test-16900294235331-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16900294235331,,1690029426287.626922efc5eebf51b8bc065f9cae11de. 2023-07-22T12:37:06,726 DEBUG [RegionOpenAndInit-ns2:test-16900294235331-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16900294235331,,1690029426287.626922efc5eebf51b8bc065f9cae11de. 2023-07-22T12:37:06,726 DEBUG [RegionOpenAndInit-ns2:test-16900294235331-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16900294235331,,1690029426287.626922efc5eebf51b8bc065f9cae11de. after waiting 0 ms 2023-07-22T12:37:06,726 DEBUG [RegionOpenAndInit-ns2:test-16900294235331-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16900294235331,,1690029426287.626922efc5eebf51b8bc065f9cae11de. 2023-07-22T12:37:06,726 INFO [RegionOpenAndInit-ns2:test-16900294235331-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16900294235331,,1690029426287.626922efc5eebf51b8bc065f9cae11de. 2023-07-22T12:37:06,726 DEBUG [RegionOpenAndInit-ns2:test-16900294235331-pool-0 {}] regionserver.HRegion(1622): Region close journal for 626922efc5eebf51b8bc065f9cae11de: 2023-07-22T12:37:06,728 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-16900294235331 execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:37:06,728 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16900294235331,,1690029426287.626922efc5eebf51b8bc065f9cae11de.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1690029426728"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029426728"}]},"ts":"1690029426728"} 2023-07-22T12:37:06,730 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:37:06,731 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-16900294235331 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:37:06,732 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16900294235331","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029426731"}]},"ts":"1690029426731"} 2023-07-22T12:37:06,733 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16900294235331, state=ENABLING in hbase:meta 2023-07-22T12:37:06,737 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16900294235331, region=626922efc5eebf51b8bc065f9cae11de, ASSIGN}] 2023-07-22T12:37:06,739 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16900294235331, region=626922efc5eebf51b8bc065f9cae11de, ASSIGN 2023-07-22T12:37:06,740 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16900294235331, region=626922efc5eebf51b8bc065f9cae11de, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,39283,1690029414077; forceNewPlan=false, retain=false 2023-07-22T12:37:06,891 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=626922efc5eebf51b8bc065f9cae11de, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:06,894 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure 626922efc5eebf51b8bc065f9cae11de, server=jenkins-hbase3.apache.org,39283,1690029414077}] 2023-07-22T12:37:06,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-07-22T12:37:07,051 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-16900294235331,,1690029426287.626922efc5eebf51b8bc065f9cae11de. 2023-07-22T12:37:07,051 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => 626922efc5eebf51b8bc065f9cae11de, NAME => 'ns2:test-16900294235331,,1690029426287.626922efc5eebf51b8bc065f9cae11de.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:37:07,052 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:37:07,052 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16900294235331 626922efc5eebf51b8bc065f9cae11de 2023-07-22T12:37:07,052 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-16900294235331,,1690029426287.626922efc5eebf51b8bc065f9cae11de.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:37:07,052 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for 626922efc5eebf51b8bc065f9cae11de 2023-07-22T12:37:07,052 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for 626922efc5eebf51b8bc065f9cae11de 2023-07-22T12:37:07,057 INFO [StoreOpener-626922efc5eebf51b8bc065f9cae11de-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 626922efc5eebf51b8bc065f9cae11de 2023-07-22T12:37:07,059 INFO [StoreOpener-626922efc5eebf51b8bc065f9cae11de-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 626922efc5eebf51b8bc065f9cae11de columnFamilyName f 2023-07-22T12:37:07,059 DEBUG [StoreOpener-626922efc5eebf51b8bc065f9cae11de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:37:07,060 INFO [StoreOpener-626922efc5eebf51b8bc065f9cae11de-1 {}] regionserver.HStore(324): Store=626922efc5eebf51b8bc065f9cae11de/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:37:07,061 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/ns2/test-16900294235331/626922efc5eebf51b8bc065f9cae11de 2023-07-22T12:37:07,061 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/ns2/test-16900294235331/626922efc5eebf51b8bc065f9cae11de 2023-07-22T12:37:07,064 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for 626922efc5eebf51b8bc065f9cae11de 2023-07-22T12:37:07,070 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/ns2/test-16900294235331/626922efc5eebf51b8bc065f9cae11de/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:37:07,071 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened 626922efc5eebf51b8bc065f9cae11de; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11539333600, jitterRate=0.07468418776988983}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:37:07,071 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for 626922efc5eebf51b8bc065f9cae11de: 2023-07-22T12:37:07,073 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2552): Post open deploy tasks for ns2:test-16900294235331,,1690029426287.626922efc5eebf51b8bc065f9cae11de., pid=18, masterSystemTime=1690029427046 2023-07-22T12:37:07,075 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2579): Finished post open deploy task for ns2:test-16900294235331,,1690029426287.626922efc5eebf51b8bc065f9cae11de. 2023-07-22T12:37:07,075 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-16900294235331,,1690029426287.626922efc5eebf51b8bc065f9cae11de. 2023-07-22T12:37:07,076 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=626922efc5eebf51b8bc065f9cae11de, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:07,081 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=18, resume processing ppid=17 2023-07-22T12:37:07,081 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure 626922efc5eebf51b8bc065f9cae11de, server=jenkins-hbase3.apache.org,39283,1690029414077 in 185 msec 2023-07-22T12:37:07,084 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=17, resume processing ppid=16 2023-07-22T12:37:07,084 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16900294235331, region=626922efc5eebf51b8bc065f9cae11de, ASSIGN in 344 msec 2023-07-22T12:37:07,085 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-16900294235331 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:37:07,085 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16900294235331","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029427085"}]},"ts":"1690029427085"} 2023-07-22T12:37:07,087 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16900294235331, state=ENABLED in hbase:meta 2023-07-22T12:37:07,092 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16900294235331 execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:37:07,094 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-16900294235331 in 805 msec 2023-07-22T12:37:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-07-22T12:37:07,401 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns2:test-16900294235331, procId: 16 completed 2023-07-22T12:37:07,405 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39283 {}] regionserver.HRegion(8210): writing data to region ns2:test-16900294235331,,1690029426287.626922efc5eebf51b8bc065f9cae11de. with WAL disabled. Data may be lost in the event of a crash. 2023-07-22T12:37:07,483 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns3:test-16900294235332', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:37:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure2.ProcedureExecutor(1033): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-16900294235332 2023-07-22T12:37:07,487 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16900294235332 execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:37:07,487 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:37:07,487 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "test-16900294235332" procId is: 19 2023-07-22T12:37:07,488 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-16900294235332 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:37:07,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:37:07,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:37:07,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:37:07,905 INFO [RegionOpenAndInit-ns3:test-16900294235332-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => c29ae4e4c3e8a42ff3c19bb2bd28710f, NAME => 'ns3:test-16900294235332,,1690029427482.c29ae4e4c3e8a42ff3c19bb2bd28710f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16900294235332', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d 2023-07-22T12:37:08,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:37:08,120 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1690029423533' 2023-07-22T12:37:08,120 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-16900294235331' 2023-07-22T12:37:08,323 DEBUG [RegionOpenAndInit-ns3:test-16900294235332-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16900294235332,,1690029427482.c29ae4e4c3e8a42ff3c19bb2bd28710f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:37:08,324 DEBUG [RegionOpenAndInit-ns3:test-16900294235332-pool-0 {}] regionserver.HRegion(1668): Closing c29ae4e4c3e8a42ff3c19bb2bd28710f, disabling compactions & flushes 2023-07-22T12:37:08,324 INFO [RegionOpenAndInit-ns3:test-16900294235332-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16900294235332,,1690029427482.c29ae4e4c3e8a42ff3c19bb2bd28710f. 2023-07-22T12:37:08,324 DEBUG [RegionOpenAndInit-ns3:test-16900294235332-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16900294235332,,1690029427482.c29ae4e4c3e8a42ff3c19bb2bd28710f. 2023-07-22T12:37:08,324 DEBUG [RegionOpenAndInit-ns3:test-16900294235332-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16900294235332,,1690029427482.c29ae4e4c3e8a42ff3c19bb2bd28710f. after waiting 0 ms 2023-07-22T12:37:08,324 DEBUG [RegionOpenAndInit-ns3:test-16900294235332-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16900294235332,,1690029427482.c29ae4e4c3e8a42ff3c19bb2bd28710f. 2023-07-22T12:37:08,324 INFO [RegionOpenAndInit-ns3:test-16900294235332-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16900294235332,,1690029427482.c29ae4e4c3e8a42ff3c19bb2bd28710f. 2023-07-22T12:37:08,324 DEBUG [RegionOpenAndInit-ns3:test-16900294235332-pool-0 {}] regionserver.HRegion(1622): Region close journal for c29ae4e4c3e8a42ff3c19bb2bd28710f: 2023-07-22T12:37:08,325 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-16900294235332 execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:37:08,326 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16900294235332,,1690029427482.c29ae4e4c3e8a42ff3c19bb2bd28710f.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1690029428325"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029428325"}]},"ts":"1690029428325"} 2023-07-22T12:37:08,328 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:37:08,329 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-16900294235332 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:37:08,329 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16900294235332","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029428329"}]},"ts":"1690029428329"} 2023-07-22T12:37:08,331 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16900294235332, state=ENABLING in hbase:meta 2023-07-22T12:37:08,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16900294235332, region=c29ae4e4c3e8a42ff3c19bb2bd28710f, ASSIGN}] 2023-07-22T12:37:08,339 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16900294235332, region=c29ae4e4c3e8a42ff3c19bb2bd28710f, ASSIGN 2023-07-22T12:37:08,340 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16900294235332, region=c29ae4e4c3e8a42ff3c19bb2bd28710f, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,39283,1690029414077; forceNewPlan=false, retain=false 2023-07-22T12:37:08,491 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=c29ae4e4c3e8a42ff3c19bb2bd28710f, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:08,494 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure c29ae4e4c3e8a42ff3c19bb2bd28710f, server=jenkins-hbase3.apache.org,39283,1690029414077}] 2023-07-22T12:37:08,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:37:08,651 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-16900294235332,,1690029427482.c29ae4e4c3e8a42ff3c19bb2bd28710f. 2023-07-22T12:37:08,652 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => c29ae4e4c3e8a42ff3c19bb2bd28710f, NAME => 'ns3:test-16900294235332,,1690029427482.c29ae4e4c3e8a42ff3c19bb2bd28710f.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:37:08,652 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:37:08,652 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16900294235332 c29ae4e4c3e8a42ff3c19bb2bd28710f 2023-07-22T12:37:08,652 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-16900294235332,,1690029427482.c29ae4e4c3e8a42ff3c19bb2bd28710f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:37:08,652 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for c29ae4e4c3e8a42ff3c19bb2bd28710f 2023-07-22T12:37:08,652 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for c29ae4e4c3e8a42ff3c19bb2bd28710f 2023-07-22T12:37:08,654 INFO [StoreOpener-c29ae4e4c3e8a42ff3c19bb2bd28710f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region c29ae4e4c3e8a42ff3c19bb2bd28710f 2023-07-22T12:37:08,656 INFO [StoreOpener-c29ae4e4c3e8a42ff3c19bb2bd28710f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c29ae4e4c3e8a42ff3c19bb2bd28710f columnFamilyName f 2023-07-22T12:37:08,656 DEBUG [StoreOpener-c29ae4e4c3e8a42ff3c19bb2bd28710f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:37:08,657 INFO [StoreOpener-c29ae4e4c3e8a42ff3c19bb2bd28710f-1 {}] regionserver.HStore(324): Store=c29ae4e4c3e8a42ff3c19bb2bd28710f/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:37:08,658 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/ns3/test-16900294235332/c29ae4e4c3e8a42ff3c19bb2bd28710f 2023-07-22T12:37:08,659 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/ns3/test-16900294235332/c29ae4e4c3e8a42ff3c19bb2bd28710f 2023-07-22T12:37:08,662 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for c29ae4e4c3e8a42ff3c19bb2bd28710f 2023-07-22T12:37:08,665 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/ns3/test-16900294235332/c29ae4e4c3e8a42ff3c19bb2bd28710f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:37:08,665 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened c29ae4e4c3e8a42ff3c19bb2bd28710f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10043737920, jitterRate=-0.0646040141582489}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:37:08,666 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for c29ae4e4c3e8a42ff3c19bb2bd28710f: 2023-07-22T12:37:08,667 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2552): Post open deploy tasks for ns3:test-16900294235332,,1690029427482.c29ae4e4c3e8a42ff3c19bb2bd28710f., pid=21, masterSystemTime=1690029428647 2023-07-22T12:37:08,669 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2579): Finished post open deploy task for ns3:test-16900294235332,,1690029427482.c29ae4e4c3e8a42ff3c19bb2bd28710f. 2023-07-22T12:37:08,669 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-16900294235332,,1690029427482.c29ae4e4c3e8a42ff3c19bb2bd28710f. 2023-07-22T12:37:08,670 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=c29ae4e4c3e8a42ff3c19bb2bd28710f, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:08,675 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=21, resume processing ppid=20 2023-07-22T12:37:08,675 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure c29ae4e4c3e8a42ff3c19bb2bd28710f, server=jenkins-hbase3.apache.org,39283,1690029414077 in 178 msec 2023-07-22T12:37:08,677 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=20, resume processing ppid=19 2023-07-22T12:37:08,677 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16900294235332, region=c29ae4e4c3e8a42ff3c19bb2bd28710f, ASSIGN in 338 msec 2023-07-22T12:37:08,678 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-16900294235332 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:37:08,679 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16900294235332","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029428678"}]},"ts":"1690029428678"} 2023-07-22T12:37:08,680 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16900294235332, state=ENABLED in hbase:meta 2023-07-22T12:37:08,683 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16900294235332 execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:37:08,685 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-16900294235332 in 1.2000 sec 2023-07-22T12:37:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:37:09,600 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns3:test-16900294235332, procId: 19 completed 2023-07-22T12:37:09,600 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-16900294235332 get assigned. Timeout = 60000ms 2023-07-22T12:37:09,603 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-07-22T12:37:09,607 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-16900294235332 assigned to meta. Checking AM states. 2023-07-22T12:37:09,608 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-07-22T12:37:09,608 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-16900294235332 assigned. 2023-07-22T12:37:09,611 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns4:test-16900294235333', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:37:09,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure2.ProcedureExecutor(1033): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-16900294235333 2023-07-22T12:37:09,615 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16900294235333 execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:37:09,615 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:37:09,615 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns4" qualifier: "test-16900294235333" procId is: 22 2023-07-22T12:37:09,616 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-16900294235333 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:37:09,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:37:09,636 INFO [RegionOpenAndInit-ns4:test-16900294235333-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 080805bc29553da1f6156bd53ce64200, NAME => 'ns4:test-16900294235333,,1690029429610.080805bc29553da1f6156bd53ce64200.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-16900294235333', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d 2023-07-22T12:37:09,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:37:09,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:37:10,059 DEBUG [RegionOpenAndInit-ns4:test-16900294235333-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-16900294235333,,1690029429610.080805bc29553da1f6156bd53ce64200.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:37:10,059 DEBUG [RegionOpenAndInit-ns4:test-16900294235333-pool-0 {}] regionserver.HRegion(1668): Closing 080805bc29553da1f6156bd53ce64200, disabling compactions & flushes 2023-07-22T12:37:10,059 INFO [RegionOpenAndInit-ns4:test-16900294235333-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-16900294235333,,1690029429610.080805bc29553da1f6156bd53ce64200. 2023-07-22T12:37:10,059 DEBUG [RegionOpenAndInit-ns4:test-16900294235333-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16900294235333,,1690029429610.080805bc29553da1f6156bd53ce64200. 2023-07-22T12:37:10,059 DEBUG [RegionOpenAndInit-ns4:test-16900294235333-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16900294235333,,1690029429610.080805bc29553da1f6156bd53ce64200. after waiting 0 ms 2023-07-22T12:37:10,059 DEBUG [RegionOpenAndInit-ns4:test-16900294235333-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16900294235333,,1690029429610.080805bc29553da1f6156bd53ce64200. 2023-07-22T12:37:10,059 INFO [RegionOpenAndInit-ns4:test-16900294235333-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-16900294235333,,1690029429610.080805bc29553da1f6156bd53ce64200. 2023-07-22T12:37:10,059 DEBUG [RegionOpenAndInit-ns4:test-16900294235333-pool-0 {}] regionserver.HRegion(1622): Region close journal for 080805bc29553da1f6156bd53ce64200: 2023-07-22T12:37:10,060 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-16900294235333 execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:37:10,061 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-16900294235333,,1690029429610.080805bc29553da1f6156bd53ce64200.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1690029430061"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029430061"}]},"ts":"1690029430061"} 2023-07-22T12:37:10,063 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:37:10,064 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-16900294235333 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:37:10,064 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16900294235333","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029430064"}]},"ts":"1690029430064"} 2023-07-22T12:37:10,066 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16900294235333, state=ENABLING in hbase:meta 2023-07-22T12:37:10,070 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16900294235333, region=080805bc29553da1f6156bd53ce64200, ASSIGN}] 2023-07-22T12:37:10,072 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16900294235333, region=080805bc29553da1f6156bd53ce64200, ASSIGN 2023-07-22T12:37:10,073 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-16900294235333, region=080805bc29553da1f6156bd53ce64200, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,39283,1690029414077; forceNewPlan=false, retain=false 2023-07-22T12:37:10,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:37:10,225 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=080805bc29553da1f6156bd53ce64200, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:10,231 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure 080805bc29553da1f6156bd53ce64200, server=jenkins-hbase3.apache.org,39283,1690029414077}] 2023-07-22T12:37:10,390 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-16900294235333,,1690029429610.080805bc29553da1f6156bd53ce64200. 2023-07-22T12:37:10,390 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => 080805bc29553da1f6156bd53ce64200, NAME => 'ns4:test-16900294235333,,1690029429610.080805bc29553da1f6156bd53ce64200.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:37:10,390 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:37:10,391 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16900294235333 080805bc29553da1f6156bd53ce64200 2023-07-22T12:37:10,391 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-16900294235333,,1690029429610.080805bc29553da1f6156bd53ce64200.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:37:10,391 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for 080805bc29553da1f6156bd53ce64200 2023-07-22T12:37:10,391 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for 080805bc29553da1f6156bd53ce64200 2023-07-22T12:37:10,393 INFO [StoreOpener-080805bc29553da1f6156bd53ce64200-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 080805bc29553da1f6156bd53ce64200 2023-07-22T12:37:10,402 INFO [StoreOpener-080805bc29553da1f6156bd53ce64200-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 080805bc29553da1f6156bd53ce64200 columnFamilyName f 2023-07-22T12:37:10,402 DEBUG [StoreOpener-080805bc29553da1f6156bd53ce64200-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:37:10,403 INFO [StoreOpener-080805bc29553da1f6156bd53ce64200-1 {}] regionserver.HStore(324): Store=080805bc29553da1f6156bd53ce64200/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:37:10,404 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/ns4/test-16900294235333/080805bc29553da1f6156bd53ce64200 2023-07-22T12:37:10,405 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/ns4/test-16900294235333/080805bc29553da1f6156bd53ce64200 2023-07-22T12:37:10,409 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for 080805bc29553da1f6156bd53ce64200 2023-07-22T12:37:10,414 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/ns4/test-16900294235333/080805bc29553da1f6156bd53ce64200/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:37:10,415 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened 080805bc29553da1f6156bd53ce64200; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11966668960, jitterRate=0.11448289453983307}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:37:10,416 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for 080805bc29553da1f6156bd53ce64200: 2023-07-22T12:37:10,417 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2552): Post open deploy tasks for ns4:test-16900294235333,,1690029429610.080805bc29553da1f6156bd53ce64200., pid=24, masterSystemTime=1690029430384 2023-07-22T12:37:10,419 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2579): Finished post open deploy task for ns4:test-16900294235333,,1690029429610.080805bc29553da1f6156bd53ce64200. 2023-07-22T12:37:10,419 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-16900294235333,,1690029429610.080805bc29553da1f6156bd53ce64200. 2023-07-22T12:37:10,420 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=080805bc29553da1f6156bd53ce64200, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:10,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=24, resume processing ppid=23 2023-07-22T12:37:10,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure 080805bc29553da1f6156bd53ce64200, server=jenkins-hbase3.apache.org,39283,1690029414077 in 192 msec 2023-07-22T12:37:10,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=23, resume processing ppid=22 2023-07-22T12:37:10,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-16900294235333, region=080805bc29553da1f6156bd53ce64200, ASSIGN in 355 msec 2023-07-22T12:37:10,429 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-16900294235333 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:37:10,429 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16900294235333","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029430429"}]},"ts":"1690029430429"} 2023-07-22T12:37:10,431 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16900294235333, state=ENABLED in hbase:meta 2023-07-22T12:37:10,434 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16900294235333 execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:37:10,436 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-16900294235333 in 823 msec 2023-07-22T12:37:10,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:37:10,725 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns4:test-16900294235333, procId: 22 completed 2023-07-22T12:37:10,725 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-16900294235333 get assigned. Timeout = 60000ms 2023-07-22T12:37:10,725 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-07-22T12:37:10,730 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-16900294235333 assigned to meta. Checking AM states. 2023-07-22T12:37:10,730 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-07-22T12:37:10,730 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-16900294235333 assigned. 2023-07-22T12:37:10,730 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x11973d96 to 127.0.0.1:51445 2023-07-22T12:37:10,730 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:37:10,731 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:37:10,846 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] hbase.ResourceChecker(147): before: backup.TestBackupMultipleDeletes#testBackupMultipleDeletes Thread=584, OpenFileDescriptor=656, MaxFileDescriptor=60000, SystemLoadAverage=271, ProcessCount=169, AvailableMemoryMB=17181 2023-07-22T12:37:10,846 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] hbase.ResourceChecker(130): Thread=584 is superior to 500 2023-07-22T12:37:10,849 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] backup.TestBackupMultipleDeletes(60): create full backup image for all tables 2023-07-22T12:37:10,850 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x548d09bd to 127.0.0.1:51445 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:37:10,864 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@396f95a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:37:10,875 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:37:10,877 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:37234, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:37:10,897 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:37:10,899 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:45678, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:37:10,910 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'backup'} 2023-07-22T12:37:10,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure2.ProcedureExecutor(1033): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-07-22T12:37:10,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=25 2023-07-22T12:37:10,925 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:37:10,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=25, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 17 msec 2023-07-22T12:37:11,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=25 2023-07-22T12:37:11,033 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:37:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure2.ProcedureExecutor(1033): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-07-22T12:37:11,038 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:37:11,038 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:37:11,038 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2023-07-22T12:37:11,039 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:37:11,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-07-22T12:37:11,060 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => b8af53427af1873ee686b4b22616f51c, NAME => 'backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d 2023-07-22T12:37:11,071 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:37:11,071 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing b8af53427af1873ee686b4b22616f51c, disabling compactions & flushes 2023-07-22T12:37:11,071 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. 2023-07-22T12:37:11,071 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. 2023-07-22T12:37:11,071 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. after waiting 0 ms 2023-07-22T12:37:11,072 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. 2023-07-22T12:37:11,072 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. 2023-07-22T12:37:11,072 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for b8af53427af1873ee686b4b22616f51c: 2023-07-22T12:37:11,074 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:37:11,074 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1690029431074"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029431074"}]},"ts":"1690029431074"} 2023-07-22T12:37:11,076 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:37:11,078 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:37:11,078 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029431078"}]},"ts":"1690029431078"} 2023-07-22T12:37:11,082 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-07-22T12:37:11,087 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=b8af53427af1873ee686b4b22616f51c, ASSIGN}] 2023-07-22T12:37:11,089 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=b8af53427af1873ee686b4b22616f51c, ASSIGN 2023-07-22T12:37:11,090 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=b8af53427af1873ee686b4b22616f51c, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,39283,1690029414077; forceNewPlan=false, retain=false 2023-07-22T12:37:11,094 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16900294235332 2023-07-22T12:37:11,094 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-16900294235332 Metrics about Tables on a single HBase RegionServer 2023-07-22T12:37:11,095 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16900294235333 2023-07-22T12:37:11,095 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-16900294235333 Metrics about Tables on a single HBase RegionServer 2023-07-22T12:37:11,096 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1690029423533 2023-07-22T12:37:11,096 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1690029423533 Metrics about Tables on a single HBase RegionServer 2023-07-22T12:37:11,097 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16900294235331 2023-07-22T12:37:11,097 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-16900294235331 Metrics about Tables on a single HBase RegionServer 2023-07-22T12:37:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-07-22T12:37:11,241 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=b8af53427af1873ee686b4b22616f51c, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:11,246 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; OpenRegionProcedure b8af53427af1873ee686b4b22616f51c, server=jenkins-hbase3.apache.org,39283,1690029414077}] 2023-07-22T12:37:11,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-07-22T12:37:11,403 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(130): Open backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. 2023-07-22T12:37:11,403 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7254): Opening region: {ENCODED => b8af53427af1873ee686b4b22616f51c, NAME => 'backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:37:11,404 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:37:11,404 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system b8af53427af1873ee686b4b22616f51c 2023-07-22T12:37:11,404 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(888): Instantiated backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:37:11,404 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7296): checking encryption for b8af53427af1873ee686b4b22616f51c 2023-07-22T12:37:11,404 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7299): checking classloading for b8af53427af1873ee686b4b22616f51c 2023-07-22T12:37:11,406 INFO [StoreOpener-b8af53427af1873ee686b4b22616f51c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region b8af53427af1873ee686b4b22616f51c 2023-07-22T12:37:11,408 INFO [StoreOpener-b8af53427af1873ee686b4b22616f51c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b8af53427af1873ee686b4b22616f51c columnFamilyName meta 2023-07-22T12:37:11,409 DEBUG [StoreOpener-b8af53427af1873ee686b4b22616f51c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:37:11,409 INFO [StoreOpener-b8af53427af1873ee686b4b22616f51c-1 {}] regionserver.HStore(324): Store=b8af53427af1873ee686b4b22616f51c/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:37:11,409 INFO [StoreOpener-b8af53427af1873ee686b4b22616f51c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region b8af53427af1873ee686b4b22616f51c 2023-07-22T12:37:11,411 INFO [StoreOpener-b8af53427af1873ee686b4b22616f51c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b8af53427af1873ee686b4b22616f51c columnFamilyName session 2023-07-22T12:37:11,411 DEBUG [StoreOpener-b8af53427af1873ee686b4b22616f51c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:37:11,411 INFO [StoreOpener-b8af53427af1873ee686b4b22616f51c-1 {}] regionserver.HStore(324): Store=b8af53427af1873ee686b4b22616f51c/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:37:11,412 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c 2023-07-22T12:37:11,413 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c 2023-07-22T12:37:11,414 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-07-22T12:37:11,416 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1076): writing seq id for b8af53427af1873ee686b4b22616f51c 2023-07-22T12:37:11,419 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:37:11,419 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): Opened b8af53427af1873ee686b4b22616f51c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11251342720, jitterRate=0.0478629469871521}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-07-22T12:37:11,420 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(992): Region open journal for b8af53427af1873ee686b4b22616f51c: 2023-07-22T12:37:11,421 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c., pid=28, masterSystemTime=1690029431398 2023-07-22T12:37:11,423 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. 2023-07-22T12:37:11,423 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(158): Opened backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. 2023-07-22T12:37:11,424 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=b8af53427af1873ee686b4b22616f51c, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:11,428 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=28, resume processing ppid=27 2023-07-22T12:37:11,428 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=28, ppid=27, state=SUCCESS; OpenRegionProcedure b8af53427af1873ee686b4b22616f51c, server=jenkins-hbase3.apache.org,39283,1690029414077 in 180 msec 2023-07-22T12:37:11,430 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=27, resume processing ppid=26 2023-07-22T12:37:11,430 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=b8af53427af1873ee686b4b22616f51c, ASSIGN in 341 msec 2023-07-22T12:37:11,431 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:37:11,431 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029431431"}]},"ts":"1690029431431"} 2023-07-22T12:37:11,433 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-07-22T12:37:11,437 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:37:11,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=26, state=SUCCESS; CreateTableProcedure table=backup:system in 404 msec 2023-07-22T12:37:11,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-07-22T12:37:11,645 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: backup:system, procId: 26 completed 2023-07-22T12:37:11,662 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:37:11,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure2.ProcedureExecutor(1033): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-07-22T12:37:11,666 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:37:11,667 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:37:11,667 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2023-07-22T12:37:11,668 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:37:11,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-07-22T12:37:11,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-07-22T12:37:11,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-07-22T12:37:12,087 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 4d15624c81ab8d3881a6ef2c42fdbab5, NAME => 'backup:system_bulk,,1690029431661.4d15624c81ab8d3881a6ef2c42fdbab5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d 2023-07-22T12:37:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-07-22T12:37:12,503 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1690029431661.4d15624c81ab8d3881a6ef2c42fdbab5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:37:12,504 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing 4d15624c81ab8d3881a6ef2c42fdbab5, disabling compactions & flushes 2023-07-22T12:37:12,504 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1690029431661.4d15624c81ab8d3881a6ef2c42fdbab5. 2023-07-22T12:37:12,504 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1690029431661.4d15624c81ab8d3881a6ef2c42fdbab5. 2023-07-22T12:37:12,504 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1690029431661.4d15624c81ab8d3881a6ef2c42fdbab5. after waiting 0 ms 2023-07-22T12:37:12,504 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1690029431661.4d15624c81ab8d3881a6ef2c42fdbab5. 2023-07-22T12:37:12,504 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1690029431661.4d15624c81ab8d3881a6ef2c42fdbab5. 2023-07-22T12:37:12,504 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for 4d15624c81ab8d3881a6ef2c42fdbab5: 2023-07-22T12:37:12,505 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:37:12,505 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1690029431661.4d15624c81ab8d3881a6ef2c42fdbab5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1690029432505"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029432505"}]},"ts":"1690029432505"} 2023-07-22T12:37:12,507 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:37:12,508 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:37:12,509 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029432508"}]},"ts":"1690029432508"} 2023-07-22T12:37:12,510 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-07-22T12:37:12,514 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=4d15624c81ab8d3881a6ef2c42fdbab5, ASSIGN}] 2023-07-22T12:37:12,516 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=4d15624c81ab8d3881a6ef2c42fdbab5, ASSIGN 2023-07-22T12:37:12,517 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=4d15624c81ab8d3881a6ef2c42fdbab5, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,39283,1690029414077; forceNewPlan=false, retain=false 2023-07-22T12:37:12,668 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=4d15624c81ab8d3881a6ef2c42fdbab5, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:12,670 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE; OpenRegionProcedure 4d15624c81ab8d3881a6ef2c42fdbab5, server=jenkins-hbase3.apache.org,39283,1690029414077}] 2023-07-22T12:37:12,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-07-22T12:37:12,826 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1690029431661.4d15624c81ab8d3881a6ef2c42fdbab5. 2023-07-22T12:37:12,826 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7254): Opening region: {ENCODED => 4d15624c81ab8d3881a6ef2c42fdbab5, NAME => 'backup:system_bulk,,1690029431661.4d15624c81ab8d3881a6ef2c42fdbab5.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:37:12,827 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:37:12,827 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk 4d15624c81ab8d3881a6ef2c42fdbab5 2023-07-22T12:37:12,827 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1690029431661.4d15624c81ab8d3881a6ef2c42fdbab5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:37:12,827 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7296): checking encryption for 4d15624c81ab8d3881a6ef2c42fdbab5 2023-07-22T12:37:12,827 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7299): checking classloading for 4d15624c81ab8d3881a6ef2c42fdbab5 2023-07-22T12:37:12,829 INFO [StoreOpener-4d15624c81ab8d3881a6ef2c42fdbab5-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 4d15624c81ab8d3881a6ef2c42fdbab5 2023-07-22T12:37:12,831 INFO [StoreOpener-4d15624c81ab8d3881a6ef2c42fdbab5-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d15624c81ab8d3881a6ef2c42fdbab5 columnFamilyName meta 2023-07-22T12:37:12,831 DEBUG [StoreOpener-4d15624c81ab8d3881a6ef2c42fdbab5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:37:12,831 INFO [StoreOpener-4d15624c81ab8d3881a6ef2c42fdbab5-1 {}] regionserver.HStore(324): Store=4d15624c81ab8d3881a6ef2c42fdbab5/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:37:12,831 INFO [StoreOpener-4d15624c81ab8d3881a6ef2c42fdbab5-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 4d15624c81ab8d3881a6ef2c42fdbab5 2023-07-22T12:37:12,833 INFO [StoreOpener-4d15624c81ab8d3881a6ef2c42fdbab5-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d15624c81ab8d3881a6ef2c42fdbab5 columnFamilyName session 2023-07-22T12:37:12,833 DEBUG [StoreOpener-4d15624c81ab8d3881a6ef2c42fdbab5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:37:12,834 INFO [StoreOpener-4d15624c81ab8d3881a6ef2c42fdbab5-1 {}] regionserver.HStore(324): Store=4d15624c81ab8d3881a6ef2c42fdbab5/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:37:12,835 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system_bulk/4d15624c81ab8d3881a6ef2c42fdbab5 2023-07-22T12:37:12,836 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system_bulk/4d15624c81ab8d3881a6ef2c42fdbab5 2023-07-22T12:37:12,837 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-07-22T12:37:12,839 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1076): writing seq id for 4d15624c81ab8d3881a6ef2c42fdbab5 2023-07-22T12:37:12,842 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system_bulk/4d15624c81ab8d3881a6ef2c42fdbab5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:37:12,843 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): Opened 4d15624c81ab8d3881a6ef2c42fdbab5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11914261920, jitterRate=0.10960210859775543}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-07-22T12:37:12,843 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(992): Region open journal for 4d15624c81ab8d3881a6ef2c42fdbab5: 2023-07-22T12:37:12,844 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system_bulk,,1690029431661.4d15624c81ab8d3881a6ef2c42fdbab5., pid=31, masterSystemTime=1690029432822 2023-07-22T12:37:12,847 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system_bulk,,1690029431661.4d15624c81ab8d3881a6ef2c42fdbab5. 2023-07-22T12:37:12,847 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1690029431661.4d15624c81ab8d3881a6ef2c42fdbab5. 2023-07-22T12:37:12,848 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=4d15624c81ab8d3881a6ef2c42fdbab5, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:12,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=31, resume processing ppid=30 2023-07-22T12:37:12,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=31, ppid=30, state=SUCCESS; OpenRegionProcedure 4d15624c81ab8d3881a6ef2c42fdbab5, server=jenkins-hbase3.apache.org,39283,1690029414077 in 180 msec 2023-07-22T12:37:12,854 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=30, resume processing ppid=29 2023-07-22T12:37:12,855 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=30, ppid=29, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=4d15624c81ab8d3881a6ef2c42fdbab5, ASSIGN in 338 msec 2023-07-22T12:37:12,855 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:37:12,856 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029432855"}]},"ts":"1690029432855"} 2023-07-22T12:37:12,857 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-07-22T12:37:12,861 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:37:12,862 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=29, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 1.1990 sec 2023-07-22T12:37:13,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-07-22T12:37:13,777 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: backup:system_bulk, procId: 29 completed 2023-07-22T12:37:13,790 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:37:13,842 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:37:13,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029433842 (current time:1690029433842). 2023-07-22T12:37:13,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:37:13,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:37:13,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x47e37ab9 to 127.0.0.1:51445 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:37:13,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bd6314c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:37:13,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:37:13,855 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:37240, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:37:13,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x47e37ab9 to 127.0.0.1:51445 2023-07-22T12:37:13,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:37:13,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:37:13,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure2.ProcedureExecutor(1033): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:37:13,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-07-22T12:37:13,871 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:37:13,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:37:13,876 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:37:13,879 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:37:13,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:37:14,122 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-07-22T12:37:14,122 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-16900294235333' 2023-07-22T12:37:14,123 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-07-22T12:37:14,123 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16900294235332' 2023-07-22T12:37:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:37:14,290 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:37:14,293 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c}] 2023-07-22T12:37:14,300 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c 2023-07-22T12:37:14,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39283 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2023-07-22T12:37:14,462 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. 2023-07-22T12:37:14,464 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2819): Flushing b8af53427af1873ee686b4b22616f51c 2/2 column families, dataSize=45 B heapSize=632 B 2023-07-22T12:37:14,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:37:14,543 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/session/e7980d1347624a0e8cf8cbac22176c14 is 49, key is activesession:/session:c/1690029433825/Put/seqid=0 2023-07-22T12:37:14,563 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/session/e7980d1347624a0e8cf8cbac22176c14 2023-07-22T12:37:14,650 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e7980d1347624a0e8cf8cbac22176c14 2023-07-22T12:37:14,654 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/session/e7980d1347624a0e8cf8cbac22176c14 as hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/e7980d1347624a0e8cf8cbac22176c14 2023-07-22T12:37:14,664 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/e7980d1347624a0e8cf8cbac22176c14, entries=1, sequenceid=5, filesize=4.9 K 2023-07-22T12:37:14,668 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for b8af53427af1873ee686b4b22616f51c in 204ms, sequenceid=5, compaction requested=false 2023-07-22T12:37:14,670 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2520): Flush status journal for b8af53427af1873ee686b4b22616f51c: 2023-07-22T12:37:14,670 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. for snapshot_backup_system completed. 2023-07-22T12:37:14,671 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:37:14,678 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:37:14,682 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-07-22T12:37:14,683 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/e7980d1347624a0e8cf8cbac22176c14] hfiles 2023-07-22T12:37:14,683 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/e7980d1347624a0e8cf8cbac22176c14 for snapshot=snapshot_backup_system 2023-07-22T12:37:14,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:37:15,100 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. 2023-07-22T12:37:15,105 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=33 2023-07-22T12:37:15,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.HMaster(3994): Remote procedure done, pid=33 2023-07-22T12:37:15,109 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region b8af53427af1873ee686b4b22616f51c 2023-07-22T12:37:15,111 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c 2023-07-22T12:37:15,115 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=33, resume processing ppid=32 2023-07-22T12:37:15,115 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:37:15,115 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=33, ppid=32, state=SUCCESS; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c in 819 msec 2023-07-22T12:37:15,117 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:37:15,121 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:37:15,121 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:37:15,125 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:37:15,551 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:37:15,564 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:37:15,565 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:37:15,894 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:37:15,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:37:16,006 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:37:16,006 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-07-22T12:37:16,008 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=32, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.1420 sec 2023-07-22T12:37:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:37:17,981 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 32 completed 2023-07-22T12:37:17,981 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(115): Backup backup_1690029430868 started at 1690029437981. 2023-07-22T12:37:18,003 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(119): Backup session backup_1690029430868 has been started. 2023-07-22T12:37:18,006 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-07-22T12:37:18,017 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-07-22T12:37:18,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-07-22T12:37:18,025 INFO [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-07-22T12:37:18,025 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-07-22T12:37:18,026 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-07-22T12:37:18,026 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-07-22T12:37:18,027 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-07-22T12:37:18,027 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:37:18,029 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:18,029 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-07-22T12:37:18,029 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:37:18,030 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:37:18,030 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:18,030 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-07-22T12:37:18,030 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-07-22T12:37:18,030 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-07-22T12:37:18,031 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-07-22T12:37:18,031 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:37:18,031 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-07-22T12:37:18,033 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-07-22T12:37:18,033 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-07-22T12:37:18,033 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-07-22T12:37:18,033 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-07-22T12:37:18,034 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-07-22T12:37:18,034 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-07-22T12:37:18,034 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,39283,1690029414077' joining acquired barrier for procedure (rolllog) in zk 2023-07-22T12:37:18,037 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:18,037 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-07-22T12:37:18,037 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:18,037 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:37:18,037 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:37:18,037 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-07-22T12:37:18,037 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-07-22T12:37:18,037 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:37:18,038 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:37:18,038 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:37:18,038 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:18,039 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:37:18,040 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,39283,1690029414077' joining acquired barrier for procedure 'rolllog' on coordinator 2023-07-22T12:37:18,040 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@3e11a7da[Count = 0] remaining members to acquire global barrier 2023-07-22T12:37:18,040 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-07-22T12:37:18,040 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-07-22T12:37:18,042 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:37:18,042 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-07-22T12:37:18,042 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-07-22T12:37:18,042 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-07-22T12:37:18,042 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:18,042 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-07-22T12:37:18,043 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-07-22T12:37:18,043 DEBUG [rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:18,043 INFO [rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1690029416315 highest: 1690029416315 on jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:18,044 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C39283%2C1690029414077:(num 1690029416315) roll requested 2023-07-22T12:37:18,055 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029438044, exclude list is [], retry=0 2023-07-22T12:37:18,060 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK] 2023-07-22T12:37:18,064 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029416315 with entries=19, filesize=4.02 KB; new WAL /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029438044 2023-07-22T12:37:18,065 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK]] 2023-07-22T12:37:18,065 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029416315 is not closed yet, will try archiving it next time 2023-07-22T12:37:18,065 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta:.meta(num 1690029417237) roll requested 2023-07-22T12:37:18,077 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029438065.meta, exclude list is [], retry=0 2023-07-22T12:37:18,081 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK] 2023-07-22T12:37:18,083 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029417237.meta with entries=36, filesize=11.13 KB; new WAL /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029438065.meta 2023-07-22T12:37:18,083 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK]] 2023-07-22T12:37:18,083 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029417237.meta is not closed yet, will try archiving it next time 2023-07-22T12:37:18,084 DEBUG [rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 41 2023-07-22T12:37:18,084 INFO [rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1690029438044 on jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:18,085 DEBUG [rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:37:18,087 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:45692, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-07-22T12:37:18,105 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-07-22T12:37:18,105 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,39283,1690029414077' in zk 2023-07-22T12:37:18,107 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:18,107 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-07-22T12:37:18,107 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:37:18,107 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-07-22T12:37:18,107 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:18,108 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:37:18,108 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:37:18,108 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:37:18,109 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:37:18,109 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:37:18,109 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:18,110 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:37:18,110 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:37:18,110 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:18,112 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,39283,1690029414077': 2023-07-22T12:37:18,112 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,39283,1690029414077' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-07-22T12:37:18,112 INFO [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-07-22T12:37:18,112 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-07-22T12:37:18,112 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-07-22T12:37:18,112 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-07-22T12:37:18,112 INFO [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-07-22T12:37:18,115 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:37:18,115 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:37:18,115 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-07-22T12:37:18,115 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-07-22T12:37:18,115 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-07-22T12:37:18,115 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:37:18,115 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:37:18,115 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:37:18,115 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:37:18,116 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:37:18,116 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:37:18,116 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:18,116 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:37:18,116 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:37:18,116 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:37:18,117 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:37:18,117 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:18,117 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:18,118 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:37:18,118 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:37:18,119 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:18,125 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:18,125 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-07-22T12:37:18,126 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-07-22T12:37:18,126 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-07-22T12:37:18,126 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-07-22T12:37:18,126 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:37:18,126 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:37:18,126 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:18,126 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:37:18,126 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:37:18,126 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:37:18,126 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:37:18,126 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-07-22T12:37:18,127 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-07-22T12:37:18,127 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:37:18,127 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:37:18,128 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin(2751): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-07-22T12:37:18,129 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin(2759): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-07-22T12:37:18,229 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin(2765): Getting current status of procedure from master... 2023-07-22T12:37:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-07-22T12:37:18,239 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1690029438238_default_test-1690029423533 table=test-1690029423533 type=FLUSH ttl=0 } 2023-07-22T12:37:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029438239 (current time:1690029438239). 2023-07-22T12:37:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:37:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1690029438238_default_test-1690029423533 VERSION not specified, setting to 2 2023-07-22T12:37:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3abc5d9c to 127.0.0.1:51445 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:37:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6778e3e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:37:18,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:37:18,249 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:37248, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:37:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3abc5d9c to 127.0.0.1:51445 2023-07-22T12:37:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:37:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:37:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure2.ProcedureExecutor(1033): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029438238_default_test-1690029423533 table=test-1690029423533 type=FLUSH ttl=0 } 2023-07-22T12:37:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1690029438238_default_test-1690029423533 table=test-1690029423533 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-07-22T12:37:18,253 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029438238_default_test-1690029423533 table=test-1690029423533 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:37:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:37:18,254 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029438238_default_test-1690029423533 table=test-1690029423533 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:37:18,256 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029438238_default_test-1690029423533 table=test-1690029423533 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:37:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:37:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:37:18,666 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029438238_default_test-1690029423533 table=test-1690029423533 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:37:18,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure dce7657497bf6b1da7900abd4083b1b8}] 2023-07-22T12:37:18,669 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure dce7657497bf6b1da7900abd4083b1b8 2023-07-22T12:37:18,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39283 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2023-07-22T12:37:18,821 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1690029423533,,1690029424029.dce7657497bf6b1da7900abd4083b1b8. 2023-07-22T12:37:18,822 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2819): Flushing dce7657497bf6b1da7900abd4083b1b8 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-07-22T12:37:18,841 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/default/test-1690029423533/dce7657497bf6b1da7900abd4083b1b8/.tmp/f/b1910e2882a7483ea0a1c393d00f3b3a is 37, key is row10/f:q1/1690029426188/Put/seqid=0 2023-07-22T12:37:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:37:19,252 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/default/test-1690029423533/dce7657497bf6b1da7900abd4083b1b8/.tmp/f/b1910e2882a7483ea0a1c393d00f3b3a 2023-07-22T12:37:19,263 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: b1910e2882a7483ea0a1c393d00f3b3a 2023-07-22T12:37:19,265 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/default/test-1690029423533/dce7657497bf6b1da7900abd4083b1b8/.tmp/f/b1910e2882a7483ea0a1c393d00f3b3a as hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/default/test-1690029423533/dce7657497bf6b1da7900abd4083b1b8/f/b1910e2882a7483ea0a1c393d00f3b3a 2023-07-22T12:37:19,274 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/default/test-1690029423533/dce7657497bf6b1da7900abd4083b1b8/f/b1910e2882a7483ea0a1c393d00f3b3a, entries=99, sequenceid=103, filesize=8.2 K 2023-07-22T12:37:19,276 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for dce7657497bf6b1da7900abd4083b1b8 in 454ms, sequenceid=103, compaction requested=false 2023-07-22T12:37:19,276 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2520): Flush status journal for dce7657497bf6b1da7900abd4083b1b8: 2023-07-22T12:37:19,276 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1690029423533,,1690029424029.dce7657497bf6b1da7900abd4083b1b8. for snapshot_1690029438238_default_test-1690029423533 completed. 2023-07-22T12:37:19,277 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(240): Storing 'test-1690029423533,,1690029424029.dce7657497bf6b1da7900abd4083b1b8.' region-info for snapshot=snapshot_1690029438238_default_test-1690029423533 2023-07-22T12:37:19,277 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:37:19,277 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/default/test-1690029423533/dce7657497bf6b1da7900abd4083b1b8/f/b1910e2882a7483ea0a1c393d00f3b3a] hfiles 2023-07-22T12:37:19,277 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/default/test-1690029423533/dce7657497bf6b1da7900abd4083b1b8/f/b1910e2882a7483ea0a1c393d00f3b3a for snapshot=snapshot_1690029438238_default_test-1690029423533 2023-07-22T12:37:19,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:37:19,692 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1690029423533,,1690029424029.dce7657497bf6b1da7900abd4083b1b8. 2023-07-22T12:37:19,692 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=35 2023-07-22T12:37:19,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.HMaster(3994): Remote procedure done, pid=35 2023-07-22T12:37:19,693 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1690029438238_default_test-1690029423533 on region dce7657497bf6b1da7900abd4083b1b8 2023-07-22T12:37:19,694 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure dce7657497bf6b1da7900abd4083b1b8 2023-07-22T12:37:19,697 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=35, resume processing ppid=34 2023-07-22T12:37:19,697 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029438238_default_test-1690029423533 table=test-1690029423533 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:37:19,698 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=35, ppid=34, state=SUCCESS; SnapshotRegionProcedure dce7657497bf6b1da7900abd4083b1b8 in 1.0280 sec 2023-07-22T12:37:19,698 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029438238_default_test-1690029423533 table=test-1690029423533 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:37:19,699 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029438238_default_test-1690029423533 table=test-1690029423533 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:37:19,699 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1690029438238_default_test-1690029423533 2023-07-22T12:37:19,700 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/.tmp/snapshot_1690029438238_default_test-1690029423533 2023-07-22T12:37:20,114 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029438238_default_test-1690029423533 table=test-1690029423533 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:37:20,121 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029438238_default_test-1690029423533 table=test-1690029423533 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:37:20,122 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/.tmp/snapshot_1690029438238_default_test-1690029423533 to hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/snapshot_1690029438238_default_test-1690029423533 2023-07-22T12:37:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:37:20,954 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029438238_default_test-1690029423533 table=test-1690029423533 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:37:20,955 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1690029438238_default_test-1690029423533 table=test-1690029423533 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-07-22T12:37:20,956 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=34, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690029438238_default_test-1690029423533 table=test-1690029423533 type=FLUSH ttl=0 } in 2.7040 sec 2023-07-22T12:37:21,094 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-07-22T12:37:21,094 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-07-22T12:37:21,095 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-07-22T12:37:21,095 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-07-22T12:37:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:37:22,362 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: default:test-1690029423533, procId: 34 completed 2023-07-22T12:37:22,363 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1690029442362_ns2_test-16900294235331 table=ns2:test-16900294235331 type=FLUSH ttl=0 } 2023-07-22T12:37:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029442363 (current time:1690029442363). 2023-07-22T12:37:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:37:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1690029442362_ns2_test-16900294235331 VERSION not specified, setting to 2 2023-07-22T12:37:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4162d4c6 to 127.0.0.1:51445 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:37:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19ea31cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:37:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:37:22,372 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56406, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:37:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4162d4c6 to 127.0.0.1:51445 2023-07-22T12:37:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:37:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:37:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure2.ProcedureExecutor(1033): Stored pid=36, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1690029442362_ns2_test-16900294235331 table=ns2:test-16900294235331 type=FLUSH ttl=0 } 2023-07-22T12:37:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1690029442362_ns2_test-16900294235331 table=ns2:test-16900294235331 type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-07-22T12:37:22,378 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1690029442362_ns2_test-16900294235331 table=ns2:test-16900294235331 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:37:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-07-22T12:37:22,379 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1690029442362_ns2_test-16900294235331 table=ns2:test-16900294235331 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:37:22,381 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1690029442362_ns2_test-16900294235331 table=ns2:test-16900294235331 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:37:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-07-22T12:37:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-07-22T12:37:22,792 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1690029442362_ns2_test-16900294235331 table=ns2:test-16900294235331 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:37:22,792 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 626922efc5eebf51b8bc065f9cae11de}] 2023-07-22T12:37:22,793 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 626922efc5eebf51b8bc065f9cae11de 2023-07-22T12:37:22,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39283 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=37 2023-07-22T12:37:22,945 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16900294235331,,1690029426287.626922efc5eebf51b8bc065f9cae11de. 2023-07-22T12:37:22,947 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2819): Flushing 626922efc5eebf51b8bc065f9cae11de 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-07-22T12:37:22,964 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/ns2/test-16900294235331/626922efc5eebf51b8bc065f9cae11de/.tmp/f/46975ccb8d55419c99d458bac3865a94 is 37, key is row10/f:q1/1690029427414/Put/seqid=0 2023-07-22T12:37:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-07-22T12:37:23,373 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/ns2/test-16900294235331/626922efc5eebf51b8bc065f9cae11de/.tmp/f/46975ccb8d55419c99d458bac3865a94 2023-07-22T12:37:23,380 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 46975ccb8d55419c99d458bac3865a94 2023-07-22T12:37:23,381 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/ns2/test-16900294235331/626922efc5eebf51b8bc065f9cae11de/.tmp/f/46975ccb8d55419c99d458bac3865a94 as hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/ns2/test-16900294235331/626922efc5eebf51b8bc065f9cae11de/f/46975ccb8d55419c99d458bac3865a94 2023-07-22T12:37:23,390 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/ns2/test-16900294235331/626922efc5eebf51b8bc065f9cae11de/f/46975ccb8d55419c99d458bac3865a94, entries=99, sequenceid=103, filesize=8.2 K 2023-07-22T12:37:23,391 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 626922efc5eebf51b8bc065f9cae11de in 444ms, sequenceid=103, compaction requested=false 2023-07-22T12:37:23,392 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2520): Flush status journal for 626922efc5eebf51b8bc065f9cae11de: 2023-07-22T12:37:23,392 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16900294235331,,1690029426287.626922efc5eebf51b8bc065f9cae11de. for snapshot_1690029442362_ns2_test-16900294235331 completed. 2023-07-22T12:37:23,392 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16900294235331,,1690029426287.626922efc5eebf51b8bc065f9cae11de.' region-info for snapshot=snapshot_1690029442362_ns2_test-16900294235331 2023-07-22T12:37:23,392 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:37:23,392 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/ns2/test-16900294235331/626922efc5eebf51b8bc065f9cae11de/f/46975ccb8d55419c99d458bac3865a94] hfiles 2023-07-22T12:37:23,392 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/ns2/test-16900294235331/626922efc5eebf51b8bc065f9cae11de/f/46975ccb8d55419c99d458bac3865a94 for snapshot=snapshot_1690029442362_ns2_test-16900294235331 2023-07-22T12:37:23,404 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16900294235331,,1690029426287.626922efc5eebf51b8bc065f9cae11de. 2023-07-22T12:37:23,404 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=37 2023-07-22T12:37:23,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.HMaster(3994): Remote procedure done, pid=37 2023-07-22T12:37:23,405 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1690029442362_ns2_test-16900294235331 on region 626922efc5eebf51b8bc065f9cae11de 2023-07-22T12:37:23,406 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 626922efc5eebf51b8bc065f9cae11de 2023-07-22T12:37:23,409 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=37, resume processing ppid=36 2023-07-22T12:37:23,409 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=37, ppid=36, state=SUCCESS; SnapshotRegionProcedure 626922efc5eebf51b8bc065f9cae11de in 615 msec 2023-07-22T12:37:23,409 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1690029442362_ns2_test-16900294235331 table=ns2:test-16900294235331 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:37:23,410 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1690029442362_ns2_test-16900294235331 table=ns2:test-16900294235331 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:37:23,411 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1690029442362_ns2_test-16900294235331 table=ns2:test-16900294235331 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:37:23,411 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1690029442362_ns2_test-16900294235331 2023-07-22T12:37:23,412 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/.tmp/snapshot_1690029442362_ns2_test-16900294235331 2023-07-22T12:37:23,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-07-22T12:37:23,829 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1690029442362_ns2_test-16900294235331 table=ns2:test-16900294235331 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:37:23,838 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1690029442362_ns2_test-16900294235331 table=ns2:test-16900294235331 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:37:23,838 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/.tmp/snapshot_1690029442362_ns2_test-16900294235331 to hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/snapshot_1690029442362_ns2_test-16900294235331 2023-07-22T12:37:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-07-22T12:37:24,665 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1690029442362_ns2_test-16900294235331 table=ns2:test-16900294235331 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:37:24,665 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1690029442362_ns2_test-16900294235331 table=ns2:test-16900294235331 type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-07-22T12:37:24,667 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=36, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1690029442362_ns2_test-16900294235331 table=ns2:test-16900294235331 type=FLUSH ttl=0 } in 2.2900 sec 2023-07-22T12:37:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-07-22T12:37:26,488 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: ns2:test-16900294235331, procId: 36 completed 2023-07-22T12:37:26,488 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1690029430868 2023-07-22T12:37:26,488 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-07-22T12:37:26,490 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-07-22T12:37:26,492 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1690029430868_test-1690029423533 2023-07-22T12:37:26,493 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1690029438238_default_test-1690029423533 to hdfs://localhost:33313/backupUT/backup_1690029430868/default/test-1690029423533/ 2023-07-22T12:37:26,502 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-07-22T12:37:26,534 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:33313, inputRoot=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d 2023-07-22T12:37:26,534 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,33617,1690029410929_-1837064904_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:33313/backupUT/backup_1690029430868/default/test-1690029423533, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:33313/backupUT/backup_1690029430868/default/test-1690029423533/.hbase-snapshot/.tmp/snapshot_1690029438238_default_test-1690029423533 2023-07-22T12:37:26,534 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:33313, inputRoot=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d, snapshotDir=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/snapshot_1690029438238_default_test-1690029423533. 2023-07-22T12:37:26,546 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/snapshot_1690029438238_default_test-1690029423533 to hdfs://localhost:33313/backupUT/backup_1690029430868/default/test-1690029423533/.hbase-snapshot/.tmp/snapshot_1690029438238_default_test-1690029423533 2023-07-22T12:37:26,987 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:26,988 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:26,989 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:26,989 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:27,980 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/hadoop-5642163327078957524.jar 2023-07-22T12:37:27,980 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:27,981 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:27,981 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:27,982 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:27,982 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:27,983 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:27,983 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:27,983 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:27,984 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:27,985 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:37:27,985 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:37:27,986 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:37:27,987 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:37:27,987 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:37:27,988 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:37:27,988 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:37:27,988 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:37:27,989 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:37:27,989 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:37:27,990 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:37:27,990 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:37:27,994 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:37:27,995 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:37:27,995 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:37:27,995 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:37:27,996 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:37:27,996 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:37:27,997 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:37:28,055 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-07-22T12:37:28,077 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1690029438238_default_test-1690029423533' hfile list 2023-07-22T12:37:28,087 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-07-22T12:37:28,317 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/mapreduce.cluster.local.dir/1690029448268/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:37:28,317 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/mapreduce.cluster.local.dir/1690029448268/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:37:28,464 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-07-22T12:37:28,522 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/default/test-1690029423533/dce7657497bf6b1da7900abd4083b1b8/f/b1910e2882a7483ea0a1c393d00f3b3a output=hdfs://localhost:33313/backupUT/backup_1690029430868/default/test-1690029423533/archive/data/default/test-1690029423533/dce7657497bf6b1da7900abd4083b1b8/f/b1910e2882a7483ea0a1c393d00f3b3a 2023-07-22T12:37:28,522 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-07-22T12:37:28,790 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-07-22T12:37:28,791 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:44718, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-07-22T12:37:29,374 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-07-22T12:37:29,375 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-07-22T12:37:29,384 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1690029438238_default_test-1690029423533 2023-07-22T12:37:29,384 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1690029438238_default_test-1690029423533 finished. 2023-07-22T12:37:29,384 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1690029430868_ns2:test-16900294235331 2023-07-22T12:37:29,384 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1690029442362_ns2_test-16900294235331 to hdfs://localhost:33313/backupUT/backup_1690029430868/ns2/test-16900294235331/ 2023-07-22T12:37:29,385 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-07-22T12:37:29,412 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:33313, inputRoot=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d 2023-07-22T12:37:29,412 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,33617,1690029410929_1403664397_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:33313/backupUT/backup_1690029430868/ns2/test-16900294235331, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:33313/backupUT/backup_1690029430868/ns2/test-16900294235331/.hbase-snapshot/.tmp/snapshot_1690029442362_ns2_test-16900294235331 2023-07-22T12:37:29,412 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:33313, inputRoot=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d, snapshotDir=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/snapshot_1690029442362_ns2_test-16900294235331. 2023-07-22T12:37:29,426 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/snapshot_1690029442362_ns2_test-16900294235331 to hdfs://localhost:33313/backupUT/backup_1690029430868/ns2/test-16900294235331/.hbase-snapshot/.tmp/snapshot_1690029442362_ns2_test-16900294235331 2023-07-22T12:37:29,861 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:29,861 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:29,862 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:29,862 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:30,856 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/hadoop-4021113748996020841.jar 2023-07-22T12:37:30,856 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:30,856 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:30,857 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:30,857 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:30,858 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:30,858 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:30,859 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:30,859 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:30,859 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:30,860 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:37:30,861 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:37:30,861 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:37:30,862 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:37:30,862 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:37:30,863 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:37:30,863 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:37:30,863 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:37:30,864 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:37:30,864 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:37:30,865 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:37:30,865 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:37:30,866 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:37:30,867 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:37:30,867 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:37:30,868 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:37:30,868 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:37:30,868 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:37:30,869 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:37:30,895 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-07-22T12:37:30,913 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1690029442362_ns2_test-16900294235331' hfile list 2023-07-22T12:37:30,915 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-07-22T12:37:30,985 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/mapreduce.cluster.local.dir/1690029450951/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:37:30,985 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/mapreduce.cluster.local.dir/1690029450951/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:37:31,046 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-07-22T12:37:31,074 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/ns2/test-16900294235331/626922efc5eebf51b8bc065f9cae11de/f/46975ccb8d55419c99d458bac3865a94 output=hdfs://localhost:33313/backupUT/backup_1690029430868/ns2/test-16900294235331/archive/data/ns2/test-16900294235331/626922efc5eebf51b8bc065f9cae11de/f/46975ccb8d55419c99d458bac3865a94 2023-07-22T12:37:31,074 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-07-22T12:37:32,024 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-07-22T12:37:32,025 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-07-22T12:37:32,034 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1690029442362_ns2_test-16900294235331 2023-07-22T12:37:32,035 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1690029442362_ns2_test-16900294235331 finished. 2023-07-22T12:37:32,035 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(1014): test-1690029423533 2023-07-22T12:37:32,035 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(1014): ns2:test-16900294235331 2023-07-22T12:37:32,110 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029430868 2023-07-22T12:37:32,110 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-07-22T12:37:32,522 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33313/backupUT/backup_1690029430868/.backup.manifest 2023-07-22T12:37:32,522 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029430868 2023-07-22T12:37:32,522 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-07-22T12:37:32,929 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33313/backupUT/backup_1690029430868/.backup.manifest 2023-07-22T12:37:32,929 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(358): Backup backup_1690029430868 finished: type=FULL,tablelist=test-1690029423533;ns2:test-16900294235331,targetRootDir=hdfs://localhost:33313/backupUT,startts=1690029437981,completets=1690029452106,bytescopied=0 2023-07-22T12:37:32,929 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-07-22T12:37:32,929 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1690029438238_default_test-1690029423533 2023-07-22T12:37:32,939 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1690029438238_default_test-1690029423533" 2023-07-22T12:37:32,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1690029438238_default_test-1690029423533 2023-07-22T12:37:32,944 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1690029438238_default_test-1690029423533 for backup backup_1690029430868 succeeded. 2023-07-22T12:37:32,944 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1690029442362_ns2_test-16900294235331 2023-07-22T12:37:32,945 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1690029442362_ns2_test-16900294235331" 2023-07-22T12:37:32,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1690029442362_ns2_test-16900294235331 2023-07-22T12:37:32,949 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1690029442362_ns2_test-16900294235331 for backup backup_1690029430868 succeeded. 2023-07-22T12:37:32,949 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:37:32,961 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:37:32,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:37:32,964 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:37:32,972 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:37:32,990 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(376): Backup backup_1690029430868 completed. 2023-07-22T12:37:33,182 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:37:33,187 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:37:33,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029453187 (current time:1690029453187). 2023-07-22T12:37:33,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:37:33,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:37:33,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x75e20db3 to 127.0.0.1:51445 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:37:33,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ee8139, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:37:33,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:37:33,196 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:41556, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:37:33,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x75e20db3 to 127.0.0.1:51445 2023-07-22T12:37:33,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:37:33,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:37:33,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure2.ProcedureExecutor(1033): Stored pid=38, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:37:33,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 38 2023-07-22T12:37:33,201 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:37:33,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-07-22T12:37:33,202 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:37:33,204 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:37:33,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-07-22T12:37:33,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-07-22T12:37:33,611 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:37:33,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c}] 2023-07-22T12:37:33,613 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=39, ppid=38, state=RUNNABLE; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c 2023-07-22T12:37:33,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39283 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=39 2023-07-22T12:37:33,765 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. 2023-07-22T12:37:33,766 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HRegion(2819): Flushing b8af53427af1873ee686b4b22616f51c 2/2 column families, dataSize=1.51 KB heapSize=2.80 KB 2023-07-22T12:37:33,780 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/meta/8b4e5904fd5541caabafed2e0a7bbecc is 171, key is trslm:hdfs://localhost:33313/backupUT\x00ns2:test-16900294235331/meta:log-roll-map/1690029452095/Put/seqid=0 2023-07-22T12:37:33,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-07-22T12:37:34,185 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=785 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/meta/8b4e5904fd5541caabafed2e0a7bbecc 2023-07-22T12:37:34,192 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8b4e5904fd5541caabafed2e0a7bbecc 2023-07-22T12:37:34,197 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/session/40fe5705ee804503bdd7677bd59e3be8 is 459, key is session:backup_1690029430868/session:context/1690029452971/Put/seqid=0 2023-07-22T12:37:34,203 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=758 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/session/40fe5705ee804503bdd7677bd59e3be8 2023-07-22T12:37:34,214 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 40fe5705ee804503bdd7677bd59e3be8 2023-07-22T12:37:34,215 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/meta/8b4e5904fd5541caabafed2e0a7bbecc as hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/8b4e5904fd5541caabafed2e0a7bbecc 2023-07-22T12:37:34,222 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/8b4e5904fd5541caabafed2e0a7bbecc, entries=6, sequenceid=17, filesize=5.8 K 2023-07-22T12:37:34,223 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/session/40fe5705ee804503bdd7677bd59e3be8 as hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/40fe5705ee804503bdd7677bd59e3be8 2023-07-22T12:37:34,233 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/40fe5705ee804503bdd7677bd59e3be8, entries=2, sequenceid=17, filesize=5.4 K 2023-07-22T12:37:34,234 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HRegion(3022): Finished flush of dataSize ~1.51 KB/1543, heapSize ~2.77 KB/2840, currentSize=0 B/0 for b8af53427af1873ee686b4b22616f51c in 469ms, sequenceid=17, compaction requested=false 2023-07-22T12:37:34,234 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HRegion(2520): Flush status journal for b8af53427af1873ee686b4b22616f51c: 2023-07-22T12:37:34,234 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. for snapshot_backup_system completed. 2023-07-22T12:37:34,234 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:37:34,235 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:37:34,235 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/8b4e5904fd5541caabafed2e0a7bbecc] hfiles 2023-07-22T12:37:34,235 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/8b4e5904fd5541caabafed2e0a7bbecc for snapshot=snapshot_backup_system 2023-07-22T12:37:34,235 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/e7980d1347624a0e8cf8cbac22176c14, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/40fe5705ee804503bdd7677bd59e3be8] hfiles 2023-07-22T12:37:34,235 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/e7980d1347624a0e8cf8cbac22176c14 for snapshot=snapshot_backup_system 2023-07-22T12:37:34,235 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/40fe5705ee804503bdd7677bd59e3be8 for snapshot=snapshot_backup_system 2023-07-22T12:37:34,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-07-22T12:37:34,642 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. 2023-07-22T12:37:34,642 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=39 2023-07-22T12:37:34,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.HMaster(3994): Remote procedure done, pid=39 2023-07-22T12:37:34,642 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region b8af53427af1873ee686b4b22616f51c 2023-07-22T12:37:34,644 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=39, ppid=38, state=RUNNABLE; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c 2023-07-22T12:37:34,646 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=39, resume processing ppid=38 2023-07-22T12:37:34,646 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:37:34,647 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=39, ppid=38, state=SUCCESS; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c in 1.0340 sec 2023-07-22T12:37:34,647 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:37:34,648 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:37:34,648 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:37:34,649 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:37:35,060 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:37:35,068 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:37:35,068 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:37:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-07-22T12:37:35,889 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:37:35,889 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 38 2023-07-22T12:37:35,890 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=38, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.6910 sec 2023-07-22T12:37:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-07-22T12:37:37,309 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 38 completed 2023-07-22T12:37:37,310 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(115): Backup backup_1690029453150 started at 1690029457310. 2023-07-22T12:37:37,311 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(119): Backup session backup_1690029453150 has been started. 2023-07-22T12:37:37,316 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalTableBackupClient(268): For incremental backup, current table set is [test-1690029423533, ns2:test-16900294235331] 2023-07-22T12:37:37,320 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(76): StartCode 1690029416315for backupID backup_1690029453150 2023-07-22T12:37:37,320 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-07-22T12:37:37,321 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-07-22T12:37:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-07-22T12:37:37,321 INFO [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-07-22T12:37:37,321 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-07-22T12:37:37,322 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-07-22T12:37:37,322 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-07-22T12:37:37,323 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-07-22T12:37:37,323 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:37:37,324 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-07-22T12:37:37,324 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:37,324 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:37:37,324 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:37:37,324 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:37,324 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-07-22T12:37:37,325 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-07-22T12:37:37,325 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-07-22T12:37:37,325 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-07-22T12:37:37,325 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:37:37,325 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-07-22T12:37:37,326 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-07-22T12:37:37,326 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-07-22T12:37:37,326 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-07-22T12:37:37,326 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-07-22T12:37:37,327 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-07-22T12:37:37,327 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-07-22T12:37:37,327 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,39283,1690029414077' joining acquired barrier for procedure (rolllog) in zk 2023-07-22T12:37:37,329 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:37,329 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-07-22T12:37:37,329 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:37,329 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:37:37,329 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:37:37,330 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-07-22T12:37:37,330 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-07-22T12:37:37,330 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:37:37,330 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:37:37,330 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:37:37,331 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:37,331 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:37:37,331 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,39283,1690029414077' joining acquired barrier for procedure 'rolllog' on coordinator 2023-07-22T12:37:37,331 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@cd7815e[Count = 0] remaining members to acquire global barrier 2023-07-22T12:37:37,331 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-07-22T12:37:37,331 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-07-22T12:37:37,332 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:37:37,333 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-07-22T12:37:37,333 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-07-22T12:37:37,333 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-07-22T12:37:37,333 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:37,333 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-07-22T12:37:37,333 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-07-22T12:37:37,333 DEBUG [rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:37,333 INFO [rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1690029438044 highest: 1690029438044 on jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:37,333 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C39283%2C1690029414077:(num 1690029438044) roll requested 2023-07-22T12:37:37,344 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029457333, exclude list is [], retry=0 2023-07-22T12:37:37,348 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK] 2023-07-22T12:37:37,350 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029438044 with entries=113, filesize=13.97 KB; new WAL /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029457333 2023-07-22T12:37:37,350 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK]] 2023-07-22T12:37:37,350 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029438044 is not closed yet, will try archiving it next time 2023-07-22T12:37:37,350 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta:.meta(num 1690029438065) roll requested 2023-07-22T12:37:37,352 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029438044 not finished, retry = 0 2023-07-22T12:37:37,361 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029457350.meta, exclude list is [], retry=0 2023-07-22T12:37:37,365 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK] 2023-07-22T12:37:37,366 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029438065.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029457350.meta 2023-07-22T12:37:37,366 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK]] 2023-07-22T12:37:37,366 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029438065.meta is not closed yet, will try archiving it next time 2023-07-22T12:37:37,368 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029438065.meta not finished, retry = 0 2023-07-22T12:37:37,373 DEBUG [rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-07-22T12:37:37,373 INFO [rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1690029457333 on jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:37,385 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-07-22T12:37:37,385 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,39283,1690029414077' in zk 2023-07-22T12:37:37,387 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:37,388 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-07-22T12:37:37,388 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:37:37,388 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:37,388 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-07-22T12:37:37,389 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:37:37,389 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:37:37,389 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:37:37,390 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:37:37,390 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:37:37,390 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:37,391 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:37:37,391 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:37:37,392 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:37,392 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,39283,1690029414077': 2023-07-22T12:37:37,392 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,39283,1690029414077' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-07-22T12:37:37,392 INFO [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-07-22T12:37:37,393 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-07-22T12:37:37,393 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-07-22T12:37:37,393 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-07-22T12:37:37,393 INFO [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-07-22T12:37:37,395 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:37:37,395 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:37:37,395 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-07-22T12:37:37,395 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-07-22T12:37:37,395 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-07-22T12:37:37,395 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:37:37,395 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:37:37,395 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:37:37,395 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:37:37,395 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:37:37,395 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:37:37,396 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:37,396 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:37:37,396 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:37:37,396 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:37:37,397 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:37:37,397 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:37,398 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:37,398 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:37:37,398 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:37:37,399 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:37,404 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:37,405 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-07-22T12:37:37,405 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-07-22T12:37:37,405 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-07-22T12:37:37,405 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-07-22T12:37:37,405 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:37:37,405 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:37:37,405 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-07-22T12:37:37,405 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:37,405 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:37:37,405 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-07-22T12:37:37,405 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:37:37,405 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:37:37,406 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:37:37,406 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin(2751): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-07-22T12:37:37,406 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:37:37,406 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin(2759): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-07-22T12:37:37,406 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:37:37,470 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029438065.meta to hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029438065.meta 2023-07-22T12:37:37,506 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin(2765): Getting current status of procedure from master... 2023-07-22T12:37:37,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-07-22T12:37:37,509 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:39283=1690029416315} newestTimestamps: {jenkins-hbase3.apache.org:39283=1690029438044} 2023-07-22T12:37:37,512 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029417237.meta 2023-07-22T12:37:37,512 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029457350.meta 2023-07-22T12:37:37,512 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029438044 2023-07-22T12:37:37,512 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029457333 2023-07-22T12:37:37,513 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029438065.meta 2023-07-22T12:37:37,924 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(146): Attempting to copy table info for:test-1690029423533 target: hdfs://localhost:33313/backupUT/backup_1690029453150/default/test-1690029423533 descriptor: 'test-1690029423533', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:37:37,924 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-07-22T12:37:37,926 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(151): Starting to write region info for table test-1690029423533 2023-07-22T12:37:38,332 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(158): Finished writing region info for table test-1690029423533 2023-07-22T12:37:38,743 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(146): Attempting to copy table info for:ns2:test-16900294235331 target: hdfs://localhost:33313/backupUT/backup_1690029453150/ns2/test-16900294235331 descriptor: 'ns2:test-16900294235331', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:37:38,744 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-07-22T12:37:38,746 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(151): Starting to write region info for table ns2:test-16900294235331 2023-07-22T12:37:39,108 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-07-22T12:37:39,108 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-07-22T12:37:39,152 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(158): Finished writing region info for table ns2:test-16900294235331 2023-07-22T12:37:39,161 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:33313/backupUT/.tmp/backup_1690029453150 from hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029438044 2023-07-22T12:37:39,164 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x25d6759e to 127.0.0.1:51445 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:37:39,170 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ec31d08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:37:39,172 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:37:39,173 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:55852, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:37:39,188 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-07-22T12:37:39,189 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1690029423533 2023-07-22T12:37:39,194 DEBUG [hconnection-0x6dd66f9b-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:37:39,196 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:41564, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:37:39,199 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1690029423533: test-1690029423533; 2023-07-22T12:37:39,199 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16900294235331 2023-07-22T12:37:39,202 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16900294235331: ns2:test-16900294235331; 2023-07-22T12:37:39,202 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(694): Configuring 2 reduce partitions to match current region count for all tables 2023-07-22T12:37:39,202 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/hbase-staging/partitions_5b9f284c-eb15-4755-b145-a4aeaa16e803 2023-07-22T12:37:39,643 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:39,644 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:39,645 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:39,645 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:40,649 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/hadoop-7253635459660145034.jar 2023-07-22T12:37:40,650 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:40,650 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:40,651 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:40,651 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:40,652 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:40,652 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:40,653 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:40,653 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:40,654 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:40,654 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:37:40,655 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:37:40,656 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:37:40,656 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:37:40,657 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:37:40,657 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:37:40,658 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:37:40,658 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:37:40,659 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:37:40,659 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:37:40,660 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:37:40,661 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:37:40,664 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:40,665 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:40,666 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:40,666 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:40,667 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:40,667 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:40,669 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:37:40,669 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: default:test-1690029423533,ns2:test-16900294235331 2023-07-22T12:37:40,669 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:37:40,669 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x25d6759e to 127.0.0.1:51445 2023-07-22T12:37:40,670 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:37:40,671 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:37:41,655 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/hadoop-7412062983891880159.jar 2023-07-22T12:37:41,688 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-07-22T12:37:41,703 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029438044 for WAL files 2023-07-22T12:37:41,704 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029438044 2023-07-22T12:37:41,785 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/mapreduce.cluster.local.dir/1690029461748/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:37:41,785 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/mapreduce.cluster.local.dir/1690029461748/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:37:41,869 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029438044 for hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029438044 (-9223372036854775808:9223372036854775807) length:14316 2023-07-22T12:37:41,888 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-07-22T12:37:41,888 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-07-22T12:37:42,029 DEBUG [pool-202-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x39987a5c to 127.0.0.1:51445 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:37:42,036 DEBUG [pool-202-thread-1 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c27b38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:37:42,039 DEBUG [hconnection-0x50f02fea-metaLookup-shared--pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:37:42,040 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:45530, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:37:42,042 DEBUG [pool-202-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x39987a5c to 127.0.0.1:51445 2023-07-22T12:37:42,043 DEBUG [pool-202-thread-1 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:37:42,045 DEBUG [pool-202-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-t10] 2023-07-22T12:37:42,045 DEBUG [pool-202-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-07-22T12:37:42,056 DEBUG [pool-202-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33313/backupUT/.tmp/backup_1690029453150/_temporary/0/_temporary/attempt_local2052492685_0003_r_000001_0/default/test-1690029423533/f/a7cefe6f6aba4c949626e58344aeadb8 is 40, key is row-t110/f:q1/1690029453024/Put/seqid=0 2023-07-22T12:37:42,059 WARN [Thread-880 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:39283] Specified favored nodes: [jenkins-hbase3.apache.org:39283] 2023-07-22T12:37:42,822 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:33313/backupUT 2023-07-22T12:37:42,822 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1690029453150 2023-07-22T12:37:42,822 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-07-22T12:37:42,841 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:33313/backupUT/.tmp/backup_1690029453150, hdfs://localhost:33313/backupUT] 2023-07-22T12:37:42,900 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-07-22T12:37:43,482 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-07-22T12:37:43,484 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1690029453150 2023-07-22T12:37:43,484 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 8797 bytes copied." 2023-07-22T12:37:43,484 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local1062645336_0004 completed: true true 2023-07-22T12:37:43,490 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=3756442 FILE: Number of bytes written=5042211 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=109861 HDFS: Number of bytes written=2325804 HDFS: Number of read operations=471 HDFS: Number of large read operations=0 HDFS: Number of write operations=207 Map-Reduce Framework Map input records=6 Map output records=0 Input split bytes=342 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1279262720 File Input Format Counters Bytes Read=1478 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=8797 Bytes Expected=8797 Files Copied=2 DIR_COPY=4 2023-07-22T12:37:43,490 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:33313/backupUT/.tmp/backup_1690029453150 to hdfs://localhost:33313/backupUT finished. 2023-07-22T12:37:43,504 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(433): deleted 0 original bulkload rows 2023-07-22T12:37:43,504 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029453150 2023-07-22T12:37:43,511 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(340): Got 1 ancestors for the current backup. 2023-07-22T12:37:43,511 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029453150 2023-07-22T12:37:43,513 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(340): Got 1 ancestors for the current backup. 2023-07-22T12:37:43,919 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33313/backupUT/backup_1690029453150/.backup.manifest 2023-07-22T12:37:43,919 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029453150 2023-07-22T12:37:43,921 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(340): Got 1 ancestors for the current backup. 2023-07-22T12:37:43,921 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029453150 2023-07-22T12:37:43,923 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(340): Got 1 ancestors for the current backup. 2023-07-22T12:37:44,329 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33313/backupUT/backup_1690029453150/.backup.manifest 2023-07-22T12:37:44,329 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029453150 2023-07-22T12:37:44,331 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(340): Got 1 ancestors for the current backup. 2023-07-22T12:37:44,738 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33313/backupUT/backup_1690029453150/.backup.manifest 2023-07-22T12:37:44,738 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(358): Backup backup_1690029453150 finished: type=INCREMENTAL,tablelist=test-1690029423533;ns2:test-16900294235331,targetRootDir=hdfs://localhost:33313/backupUT,startts=1690029457310,completets=1690029463504,bytescopied=0 2023-07-22T12:37:44,738 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:37:44,744 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:37:44,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:37:44,748 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:37:44,750 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:37:44,753 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(376): Backup backup_1690029453150 completed. 2023-07-22T12:37:44,886 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:37:44,889 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:37:44,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029464889 (current time:1690029464889). 2023-07-22T12:37:44,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:37:44,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:37:44,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x08aecdfd to 127.0.0.1:51445 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:37:44,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@172ec89a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:37:44,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:37:44,899 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:45546, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:37:44,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x08aecdfd to 127.0.0.1:51445 2023-07-22T12:37:44,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:37:44,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:37:44,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure2.ProcedureExecutor(1033): Stored pid=40, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:37:44,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 40 2023-07-22T12:37:44,903 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:37:44,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=40 2023-07-22T12:37:44,904 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:37:44,905 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:37:45,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=40 2023-07-22T12:37:45,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=40 2023-07-22T12:37:45,312 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:37:45,312 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c}] 2023-07-22T12:37:45,314 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=41, ppid=40, state=RUNNABLE; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c 2023-07-22T12:37:45,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39283 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=41 2023-07-22T12:37:45,466 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. 2023-07-22T12:37:45,466 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.HRegion(2819): Flushing b8af53427af1873ee686b4b22616f51c 2/2 column families, dataSize=1.37 KB heapSize=2.52 KB 2023-07-22T12:37:45,470 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/meta/d7524c21e0884ef7a726165d97bdf81e is 171, key is trslm:hdfs://localhost:33313/backupUT\x00ns2:test-16900294235331/meta:log-roll-map/1690029463495/Put/seqid=0 2023-07-22T12:37:45,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=40 2023-07-22T12:37:45,875 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=531 B at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/meta/d7524c21e0884ef7a726165d97bdf81e 2023-07-22T12:37:45,881 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: d7524c21e0884ef7a726165d97bdf81e 2023-07-22T12:37:45,884 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/session/882f60fdcf9248459ac9b39b6e8e1f01 is 360, key is session:backup_1690029453150/session:context/1690029464748/Put/seqid=0 2023-07-22T12:37:46,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=40 2023-07-22T12:37:46,289 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=873 B at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/session/882f60fdcf9248459ac9b39b6e8e1f01 2023-07-22T12:37:46,295 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 882f60fdcf9248459ac9b39b6e8e1f01 2023-07-22T12:37:46,296 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/meta/d7524c21e0884ef7a726165d97bdf81e as hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/d7524c21e0884ef7a726165d97bdf81e 2023-07-22T12:37:46,302 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/d7524c21e0884ef7a726165d97bdf81e, entries=4, sequenceid=28, filesize=5.7 K 2023-07-22T12:37:46,303 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/session/882f60fdcf9248459ac9b39b6e8e1f01 as hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/882f60fdcf9248459ac9b39b6e8e1f01 2023-07-22T12:37:46,310 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/882f60fdcf9248459ac9b39b6e8e1f01, entries=2, sequenceid=28, filesize=5.3 K 2023-07-22T12:37:46,311 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.HRegion(3022): Finished flush of dataSize ~1.37 KB/1404, heapSize ~2.49 KB/2552, currentSize=0 B/0 for b8af53427af1873ee686b4b22616f51c in 845ms, sequenceid=28, compaction requested=true 2023-07-22T12:37:46,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.HRegion(2520): Flush status journal for b8af53427af1873ee686b4b22616f51c: 2023-07-22T12:37:46,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. for snapshot_backup_system completed. 2023-07-22T12:37:46,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:37:46,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:37:46,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/8b4e5904fd5541caabafed2e0a7bbecc, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/d7524c21e0884ef7a726165d97bdf81e] hfiles 2023-07-22T12:37:46,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/8b4e5904fd5541caabafed2e0a7bbecc for snapshot=snapshot_backup_system 2023-07-22T12:37:46,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/d7524c21e0884ef7a726165d97bdf81e for snapshot=snapshot_backup_system 2023-07-22T12:37:46,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/e7980d1347624a0e8cf8cbac22176c14, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/40fe5705ee804503bdd7677bd59e3be8, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/882f60fdcf9248459ac9b39b6e8e1f01] hfiles 2023-07-22T12:37:46,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/e7980d1347624a0e8cf8cbac22176c14 for snapshot=snapshot_backup_system 2023-07-22T12:37:46,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/40fe5705ee804503bdd7677bd59e3be8 for snapshot=snapshot_backup_system 2023-07-22T12:37:46,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/882f60fdcf9248459ac9b39b6e8e1f01 for snapshot=snapshot_backup_system 2023-07-22T12:37:46,718 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. 2023-07-22T12:37:46,718 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=41 2023-07-22T12:37:46,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.HMaster(3994): Remote procedure done, pid=41 2023-07-22T12:37:46,718 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region b8af53427af1873ee686b4b22616f51c 2023-07-22T12:37:46,720 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=41, ppid=40, state=RUNNABLE; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c 2023-07-22T12:37:46,722 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=41, resume processing ppid=40 2023-07-22T12:37:46,722 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:37:46,722 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=41, ppid=40, state=SUCCESS; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c in 1.4090 sec 2023-07-22T12:37:46,723 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:37:46,724 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:37:46,724 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:37:46,725 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:37:47,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=40 2023-07-22T12:37:47,136 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:37:47,145 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:37:47,146 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:37:47,965 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:37:47,965 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 40 2023-07-22T12:37:47,966 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=40, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0650 sec 2023-07-22T12:37:49,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=40 2023-07-22T12:37:49,010 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 40 completed 2023-07-22T12:37:49,010 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(115): Backup backup_1690029464859 started at 1690029469010. 2023-07-22T12:37:49,011 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(119): Backup session backup_1690029464859 has been started. 2023-07-22T12:37:49,013 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalTableBackupClient(268): For incremental backup, current table set is [test-1690029423533, ns2:test-16900294235331] 2023-07-22T12:37:49,020 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(76): StartCode 1690029438044for backupID backup_1690029464859 2023-07-22T12:37:49,020 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-07-22T12:37:49,021 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-07-22T12:37:49,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-07-22T12:37:49,022 INFO [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-07-22T12:37:49,022 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-07-22T12:37:49,023 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-07-22T12:37:49,023 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-07-22T12:37:49,024 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-07-22T12:37:49,024 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:37:49,025 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:49,025 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-07-22T12:37:49,025 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:37:49,025 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:37:49,025 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:49,025 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-07-22T12:37:49,025 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-07-22T12:37:49,026 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-07-22T12:37:49,026 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-07-22T12:37:49,026 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:37:49,026 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-07-22T12:37:49,027 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-07-22T12:37:49,027 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-07-22T12:37:49,027 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-07-22T12:37:49,027 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-07-22T12:37:49,028 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-07-22T12:37:49,028 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-07-22T12:37:49,028 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,39283,1690029414077' joining acquired barrier for procedure (rolllog) in zk 2023-07-22T12:37:49,037 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-07-22T12:37:49,037 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:49,037 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:49,037 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:37:49,037 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:37:49,037 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-07-22T12:37:49,037 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-07-22T12:37:49,037 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:37:49,037 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:37:49,038 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:37:49,038 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:49,038 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:37:49,039 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,39283,1690029414077' joining acquired barrier for procedure 'rolllog' on coordinator 2023-07-22T12:37:49,039 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@1aef9468[Count = 0] remaining members to acquire global barrier 2023-07-22T12:37:49,039 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-07-22T12:37:49,039 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-07-22T12:37:49,040 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:37:49,040 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-07-22T12:37:49,040 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-07-22T12:37:49,040 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-07-22T12:37:49,040 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:49,040 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-07-22T12:37:49,040 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-07-22T12:37:49,040 DEBUG [rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:49,040 INFO [rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1690029457333 highest: 1690029457333 on jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:49,041 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C39283%2C1690029414077:(num 1690029457333) roll requested 2023-07-22T12:37:49,052 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029469041, exclude list is [], retry=0 2023-07-22T12:37:49,058 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK] 2023-07-22T12:37:49,060 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029457333 with entries=109, filesize=13.33 KB; new WAL /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029469041 2023-07-22T12:37:49,061 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK]] 2023-07-22T12:37:49,061 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029457333 is not closed yet, will try archiving it next time 2023-07-22T12:37:49,061 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta:.meta(num 1690029457350) roll requested 2023-07-22T12:37:49,072 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029469061.meta, exclude list is [], retry=0 2023-07-22T12:37:49,076 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK] 2023-07-22T12:37:49,077 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029457350.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029469061.meta 2023-07-22T12:37:49,077 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK]] 2023-07-22T12:37:49,077 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029457350.meta is not closed yet, will try archiving it next time 2023-07-22T12:37:49,079 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029457350.meta not finished, retry = 0 2023-07-22T12:37:49,081 DEBUG [rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-07-22T12:37:49,081 INFO [rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1690029469041 on jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:49,090 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-07-22T12:37:49,090 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,39283,1690029414077' in zk 2023-07-22T12:37:49,093 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:49,093 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-07-22T12:37:49,093 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:37:49,093 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-07-22T12:37:49,094 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:49,094 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:37:49,094 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:37:49,095 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:37:49,095 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:37:49,095 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:37:49,095 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:49,096 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:37:49,096 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:37:49,096 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:49,097 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,39283,1690029414077': 2023-07-22T12:37:49,097 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,39283,1690029414077' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-07-22T12:37:49,097 INFO [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-07-22T12:37:49,097 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-07-22T12:37:49,097 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-07-22T12:37:49,097 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-07-22T12:37:49,097 INFO [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-07-22T12:37:49,099 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:37:49,099 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:37:49,099 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-07-22T12:37:49,099 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-07-22T12:37:49,099 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-07-22T12:37:49,099 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:37:49,100 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:37:49,100 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:37:49,100 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:37:49,100 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:37:49,100 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:37:49,100 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:49,100 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:37:49,100 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:37:49,101 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:37:49,101 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:37:49,101 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:49,101 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:49,102 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:37:49,102 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:37:49,103 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:49,106 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-07-22T12:37:49,106 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:49,106 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-07-22T12:37:49,106 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:37:49,106 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-07-22T12:37:49,106 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-07-22T12:37:49,106 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:37:49,106 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:37:49,106 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-07-22T12:37:49,106 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:37:49,106 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-07-22T12:37:49,106 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:37:49,106 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:37:49,106 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:37:49,106 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2023-07-22T12:37:49,107 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin(2751): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-07-22T12:37:49,107 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin(2759): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-07-22T12:37:49,107 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:37:49,107 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:37:49,180 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029457350.meta to hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029457350.meta 2023-07-22T12:37:49,207 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin(2765): Getting current status of procedure from master... 2023-07-22T12:37:49,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-07-22T12:37:49,211 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:39283=1690029438044} newestTimestamps: {jenkins-hbase3.apache.org:39283=1690029457333} 2023-07-22T12:37:49,212 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029417237.meta 2023-07-22T12:37:49,212 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029469061.meta 2023-07-22T12:37:49,212 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029457333 2023-07-22T12:37:49,212 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029469041 2023-07-22T12:37:49,213 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029438065.meta 2023-07-22T12:37:49,213 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029457350.meta 2023-07-22T12:37:49,625 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(146): Attempting to copy table info for:test-1690029423533 target: hdfs://localhost:33313/backupUT/backup_1690029464859/default/test-1690029423533 descriptor: 'test-1690029423533', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:37:49,625 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-07-22T12:37:49,628 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(151): Starting to write region info for table test-1690029423533 2023-07-22T12:37:50,033 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(158): Finished writing region info for table test-1690029423533 2023-07-22T12:37:50,445 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(146): Attempting to copy table info for:ns2:test-16900294235331 target: hdfs://localhost:33313/backupUT/backup_1690029464859/ns2/test-16900294235331 descriptor: 'ns2:test-16900294235331', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:37:50,445 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-07-22T12:37:50,448 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(151): Starting to write region info for table ns2:test-16900294235331 2023-07-22T12:37:50,854 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(158): Finished writing region info for table ns2:test-16900294235331 2023-07-22T12:37:50,859 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:33313/backupUT/.tmp/backup_1690029464859 from hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029457333 2023-07-22T12:37:50,859 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x62cab35e to 127.0.0.1:51445 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:37:50,865 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cb1ee11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:37:50,866 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:37:50,868 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:59564, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:37:50,870 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-07-22T12:37:50,870 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1690029423533 2023-07-22T12:37:50,874 DEBUG [hconnection-0x3f0ecea-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:37:50,875 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:38200, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:37:50,876 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1690029423533: test-1690029423533; 2023-07-22T12:37:50,876 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16900294235331 2023-07-22T12:37:50,879 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16900294235331: ns2:test-16900294235331; 2023-07-22T12:37:50,879 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(694): Configuring 2 reduce partitions to match current region count for all tables 2023-07-22T12:37:50,879 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/hbase-staging/partitions_53b93b9e-64d8-42dd-9ca8-bf0fb1264a9f 2023-07-22T12:37:51,286 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:51,286 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:51,287 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:51,287 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:52,270 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/hadoop-5196094976238590081.jar 2023-07-22T12:37:52,271 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:52,271 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:52,271 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:52,272 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:52,272 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:52,273 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:52,273 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:52,273 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:52,274 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:52,274 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:37:52,275 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:37:52,275 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:37:52,276 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:37:52,276 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:37:52,277 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:37:52,277 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:37:52,277 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:37:52,278 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:37:52,278 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:37:52,279 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:37:52,279 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:37:52,279 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:52,280 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:52,280 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:52,280 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:52,281 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:52,281 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:37:52,282 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:37:52,282 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: default:test-1690029423533,ns2:test-16900294235331 2023-07-22T12:37:52,282 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:37:52,282 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x62cab35e to 127.0.0.1:51445 2023-07-22T12:37:52,282 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:37:52,283 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:37:53,287 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/hadoop-1400822554365432986.jar 2023-07-22T12:37:53,293 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-07-22T12:37:53,309 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029457333 for WAL files 2023-07-22T12:37:53,310 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029457333 2023-07-22T12:37:53,393 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/mapreduce.cluster.local.dir/1690029473356/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:37:53,393 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/mapreduce.cluster.local.dir/1690029473356/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:37:53,455 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029457333 for hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029457333 (-9223372036854775808:9223372036854775807) length:13653 2023-07-22T12:37:53,462 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-07-22T12:37:53,462 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-07-22T12:37:53,476 DEBUG [pool-215-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2b81187d to 127.0.0.1:51445 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:37:53,482 DEBUG [pool-215-thread-1 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a7e5bc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:37:53,484 DEBUG [hconnection-0x5e569d81-metaLookup-shared--pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:37:53,485 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:38208, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:37:53,487 DEBUG [pool-215-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2b81187d to 127.0.0.1:51445 2023-07-22T12:37:53,487 DEBUG [pool-215-thread-1 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:37:53,488 DEBUG [pool-215-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-t20] 2023-07-22T12:37:53,488 DEBUG [pool-215-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-07-22T12:37:53,496 DEBUG [pool-215-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33313/backupUT/.tmp/backup_1690029464859/_temporary/0/_temporary/attempt_local773684710_0005_r_000000_0/ns2/test-16900294235331/f/5927560aa224447088518255d2bb5a02 is 40, key is row-t210/f:q1/1690029464778/Put/seqid=0 2023-07-22T12:37:53,498 WARN [Thread-1013 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:39283] Specified favored nodes: [jenkins-hbase3.apache.org:39283] 2023-07-22T12:37:54,442 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:33313/backupUT 2023-07-22T12:37:54,442 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1690029464859 2023-07-22T12:37:54,442 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-07-22T12:37:54,447 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:33313/backupUT/.tmp/backup_1690029464859, hdfs://localhost:33313/backupUT] 2023-07-22T12:37:54,474 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-07-22T12:37:55,077 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-07-22T12:37:55,079 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1690029464859 2023-07-22T12:37:55,079 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 9113 bytes copied." 2023-07-22T12:37:55,080 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local1409706317_0006 completed: true true 2023-07-22T12:37:55,083 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=4386262 FILE: Number of bytes written=7440562 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=157493 HDFS: Number of bytes written=2359715 HDFS: Number of read operations=599 HDFS: Number of large read operations=0 HDFS: Number of write operations=254 Map-Reduce Framework Map input records=6 Map output records=0 Input split bytes=341 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1355808768 File Input Format Counters Bytes Read=1452 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=9113 Bytes Expected=9113 Files Copied=2 DIR_COPY=4 2023-07-22T12:37:55,083 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:33313/backupUT/.tmp/backup_1690029464859 to hdfs://localhost:33313/backupUT finished. 2023-07-22T12:37:55,093 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(433): deleted 0 original bulkload rows 2023-07-22T12:37:55,093 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029464859 2023-07-22T12:37:55,096 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029453150 to construct the dependency. 2023-07-22T12:37:55,096 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029453150 2023-07-22T12:37:55,099 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029453150/.backup.manifest 2023-07-22T12:37:55,099 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029453150BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:37:55,099 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(340): Got 2 ancestors for the current backup. 2023-07-22T12:37:55,099 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029464859 2023-07-22T12:37:55,101 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029453150 to construct the dependency. 2023-07-22T12:37:55,101 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029453150 2023-07-22T12:37:55,103 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029453150/.backup.manifest 2023-07-22T12:37:55,103 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029453150BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:37:55,103 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(340): Got 2 ancestors for the current backup. 2023-07-22T12:37:55,508 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33313/backupUT/backup_1690029464859/.backup.manifest 2023-07-22T12:37:55,508 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029464859 2023-07-22T12:37:55,510 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029453150 to construct the dependency. 2023-07-22T12:37:55,510 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029453150 2023-07-22T12:37:55,513 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029453150/.backup.manifest 2023-07-22T12:37:55,513 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029453150BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:37:55,513 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(340): Got 2 ancestors for the current backup. 2023-07-22T12:37:55,513 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029464859 2023-07-22T12:37:55,515 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029453150 to construct the dependency. 2023-07-22T12:37:55,515 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029453150 2023-07-22T12:37:55,517 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029453150/.backup.manifest 2023-07-22T12:37:55,517 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029453150BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:37:55,517 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(340): Got 2 ancestors for the current backup. 2023-07-22T12:37:55,923 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33313/backupUT/backup_1690029464859/.backup.manifest 2023-07-22T12:37:55,923 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029464859 2023-07-22T12:37:55,925 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029453150 to construct the dependency. 2023-07-22T12:37:55,925 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029453150 2023-07-22T12:37:55,927 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029453150/.backup.manifest 2023-07-22T12:37:55,927 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029453150BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:37:55,927 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(340): Got 2 ancestors for the current backup. 2023-07-22T12:37:56,333 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33313/backupUT/backup_1690029464859/.backup.manifest 2023-07-22T12:37:56,333 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(358): Backup backup_1690029464859 finished: type=INCREMENTAL,tablelist=test-1690029423533;ns2:test-16900294235331,targetRootDir=hdfs://localhost:33313/backupUT,startts=1690029469010,completets=1690029475093,bytescopied=0 2023-07-22T12:37:56,333 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:37:56,338 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:37:56,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:37:56,341 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:37:56,343 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:37:56,346 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(376): Backup backup_1690029464859 completed. 2023-07-22T12:37:56,453 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:37:56,456 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:37:56,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029476456 (current time:1690029476456). 2023-07-22T12:37:56,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:37:56,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:37:56,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x50196934 to 127.0.0.1:51445 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:37:56,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5208dbf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:37:56,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:37:56,465 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:38224, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:37:56,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x50196934 to 127.0.0.1:51445 2023-07-22T12:37:56,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:37:56,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:37:56,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure2.ProcedureExecutor(1033): Stored pid=42, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:37:56,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 42 2023-07-22T12:37:56,468 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:37:56,469 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:37:56,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-07-22T12:37:56,471 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:37:56,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-07-22T12:37:56,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-07-22T12:37:56,879 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:37:56,879 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c}] 2023-07-22T12:37:56,881 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c 2023-07-22T12:37:57,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39283 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=43 2023-07-22T12:37:57,032 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. 2023-07-22T12:37:57,033 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(2819): Flushing b8af53427af1873ee686b4b22616f51c 2/2 column families, dataSize=1.37 KB heapSize=2.52 KB 2023-07-22T12:37:57,039 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/meta/4b24f1898bb44138bbfed9c444ed2091 is 171, key is trslm:hdfs://localhost:33313/backupUT\x00ns2:test-16900294235331/meta:log-roll-map/1690029475087/Put/seqid=0 2023-07-22T12:37:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-07-22T12:37:57,443 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=531 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/meta/4b24f1898bb44138bbfed9c444ed2091 2023-07-22T12:37:57,450 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 4b24f1898bb44138bbfed9c444ed2091 2023-07-22T12:37:57,453 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/session/43bbdf59d84f4c98ac7bd38ac51d7e69 is 360, key is session:backup_1690029464859/session:context/1690029476342/Put/seqid=0 2023-07-22T12:37:57,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-07-22T12:37:57,858 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=873 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/session/43bbdf59d84f4c98ac7bd38ac51d7e69 2023-07-22T12:37:57,864 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 43bbdf59d84f4c98ac7bd38ac51d7e69 2023-07-22T12:37:57,865 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/meta/4b24f1898bb44138bbfed9c444ed2091 as hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/4b24f1898bb44138bbfed9c444ed2091 2023-07-22T12:37:57,870 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/4b24f1898bb44138bbfed9c444ed2091, entries=4, sequenceid=39, filesize=5.7 K 2023-07-22T12:37:57,871 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/session/43bbdf59d84f4c98ac7bd38ac51d7e69 as hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/43bbdf59d84f4c98ac7bd38ac51d7e69 2023-07-22T12:37:57,877 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/43bbdf59d84f4c98ac7bd38ac51d7e69, entries=2, sequenceid=39, filesize=5.3 K 2023-07-22T12:37:57,878 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(3022): Finished flush of dataSize ~1.37 KB/1404, heapSize ~2.49 KB/2552, currentSize=0 B/0 for b8af53427af1873ee686b4b22616f51c in 845ms, sequenceid=39, compaction requested=true 2023-07-22T12:37:57,878 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(2520): Flush status journal for b8af53427af1873ee686b4b22616f51c: 2023-07-22T12:37:57,878 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. for snapshot_backup_system completed. 2023-07-22T12:37:57,878 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:37:57,878 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:37:57,878 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/8b4e5904fd5541caabafed2e0a7bbecc, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/d7524c21e0884ef7a726165d97bdf81e, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/4b24f1898bb44138bbfed9c444ed2091] hfiles 2023-07-22T12:37:57,878 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/8b4e5904fd5541caabafed2e0a7bbecc for snapshot=snapshot_backup_system 2023-07-22T12:37:57,878 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/d7524c21e0884ef7a726165d97bdf81e for snapshot=snapshot_backup_system 2023-07-22T12:37:57,878 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/4b24f1898bb44138bbfed9c444ed2091 for snapshot=snapshot_backup_system 2023-07-22T12:37:57,878 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/e7980d1347624a0e8cf8cbac22176c14, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/40fe5705ee804503bdd7677bd59e3be8, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/882f60fdcf9248459ac9b39b6e8e1f01, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/43bbdf59d84f4c98ac7bd38ac51d7e69] hfiles 2023-07-22T12:37:57,878 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/e7980d1347624a0e8cf8cbac22176c14 for snapshot=snapshot_backup_system 2023-07-22T12:37:57,878 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/40fe5705ee804503bdd7677bd59e3be8 for snapshot=snapshot_backup_system 2023-07-22T12:37:57,878 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/882f60fdcf9248459ac9b39b6e8e1f01 for snapshot=snapshot_backup_system 2023-07-22T12:37:57,878 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/43bbdf59d84f4c98ac7bd38ac51d7e69 for snapshot=snapshot_backup_system 2023-07-22T12:37:58,284 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. 2023-07-22T12:37:58,284 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=43 2023-07-22T12:37:58,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.HMaster(3994): Remote procedure done, pid=43 2023-07-22T12:37:58,284 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region b8af53427af1873ee686b4b22616f51c 2023-07-22T12:37:58,286 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c 2023-07-22T12:37:58,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=43, resume processing ppid=42 2023-07-22T12:37:58,288 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:37:58,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=43, ppid=42, state=SUCCESS; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c in 1.4070 sec 2023-07-22T12:37:58,289 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:37:58,289 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:37:58,289 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:37:58,290 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:37:58,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-07-22T12:37:58,700 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:37:58,709 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:37:58,709 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:37:59,195 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region e1879fe3670023eba374a8cb2f79a944 changed from -1.0 to 0.0, refreshing cache 2023-07-22T12:37:59,528 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:37:59,528 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 42 2023-07-22T12:37:59,529 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=42, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0620 sec 2023-07-22T12:38:00,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-07-22T12:38:00,577 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 42 completed 2023-07-22T12:38:00,577 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(115): Backup backup_1690029476431 started at 1690029480577. 2023-07-22T12:38:00,579 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(119): Backup session backup_1690029476431 has been started. 2023-07-22T12:38:00,580 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalTableBackupClient(268): For incremental backup, current table set is [test-1690029423533, ns2:test-16900294235331] 2023-07-22T12:38:00,587 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(76): StartCode 1690029457333for backupID backup_1690029476431 2023-07-22T12:38:00,587 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-07-22T12:38:00,588 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-07-22T12:38:00,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-07-22T12:38:00,588 INFO [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-07-22T12:38:00,589 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-07-22T12:38:00,589 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-07-22T12:38:00,589 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-07-22T12:38:00,590 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-07-22T12:38:00,590 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:38:00,592 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-07-22T12:38:00,592 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:00,593 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:38:00,593 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:38:00,593 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:00,593 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-07-22T12:38:00,593 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-07-22T12:38:00,594 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-07-22T12:38:00,594 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-07-22T12:38:00,594 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:38:00,594 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-07-22T12:38:00,595 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-07-22T12:38:00,595 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-07-22T12:38:00,595 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-07-22T12:38:00,595 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-07-22T12:38:00,595 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-07-22T12:38:00,595 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-07-22T12:38:00,596 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,39283,1690029414077' joining acquired barrier for procedure (rolllog) in zk 2023-07-22T12:38:00,598 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:00,598 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-07-22T12:38:00,598 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:00,598 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:38:00,598 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:38:00,598 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-07-22T12:38:00,598 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-07-22T12:38:00,598 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:38:00,599 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:38:00,599 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:38:00,599 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:00,600 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:38:00,600 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,39283,1690029414077' joining acquired barrier for procedure 'rolllog' on coordinator 2023-07-22T12:38:00,600 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-07-22T12:38:00,600 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@13c65c9e[Count = 0] remaining members to acquire global barrier 2023-07-22T12:38:00,600 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-07-22T12:38:00,601 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:38:00,601 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-07-22T12:38:00,601 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-07-22T12:38:00,601 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-07-22T12:38:00,602 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-07-22T12:38:00,602 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:00,602 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-07-22T12:38:00,602 DEBUG [rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:00,602 INFO [rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1690029469041 highest: 1690029469041 on jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:00,602 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C39283%2C1690029414077:(num 1690029469041) roll requested 2023-07-22T12:38:00,621 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029480602, exclude list is [], retry=0 2023-07-22T12:38:00,626 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK] 2023-07-22T12:38:00,628 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029469041 with entries=109, filesize=13.07 KB; new WAL /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029480602 2023-07-22T12:38:00,628 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK]] 2023-07-22T12:38:00,628 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029469041 is not closed yet, will try archiving it next time 2023-07-22T12:38:00,628 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta:.meta(num 1690029469061) roll requested 2023-07-22T12:38:00,642 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029480628.meta, exclude list is [], retry=0 2023-07-22T12:38:00,646 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK] 2023-07-22T12:38:00,647 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029469061.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029480628.meta 2023-07-22T12:38:00,647 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK]] 2023-07-22T12:38:00,647 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029469061.meta is not closed yet, will try archiving it next time 2023-07-22T12:38:00,649 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029469061.meta to hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029469061.meta 2023-07-22T12:38:00,662 DEBUG [rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2023-07-22T12:38:00,662 INFO [rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1690029480602 on jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:00,671 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-07-22T12:38:00,671 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,39283,1690029414077' in zk 2023-07-22T12:38:00,674 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:00,674 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-07-22T12:38:00,674 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:38:00,674 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-07-22T12:38:00,675 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:00,675 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:38:00,675 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:38:00,675 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:38:00,676 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:38:00,676 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:38:00,676 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:00,677 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:38:00,677 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:38:00,677 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:00,678 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,39283,1690029414077': 2023-07-22T12:38:00,678 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,39283,1690029414077' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-07-22T12:38:00,678 INFO [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-07-22T12:38:00,678 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-07-22T12:38:00,678 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-07-22T12:38:00,678 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-07-22T12:38:00,678 INFO [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-07-22T12:38:00,681 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:38:00,681 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:38:00,681 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-07-22T12:38:00,681 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-07-22T12:38:00,681 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-07-22T12:38:00,681 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:38:00,681 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:38:00,681 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:38:00,681 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:38:00,681 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:38:00,681 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:38:00,681 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:00,682 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:38:00,682 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:38:00,682 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:38:00,682 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:38:00,683 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:00,683 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:00,683 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:38:00,683 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:38:00,684 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:00,687 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:00,687 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-07-22T12:38:00,687 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-07-22T12:38:00,687 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-07-22T12:38:00,687 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:00,687 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-07-22T12:38:00,687 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:38:00,687 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:38:00,687 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:38:00,688 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:38:00,688 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:38:00,692 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:38:00,692 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-07-22T12:38:00,692 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-07-22T12:38:00,692 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:38:00,692 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:38:00,693 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin(2751): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-07-22T12:38:00,693 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin(2759): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-07-22T12:38:00,793 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin(2765): Getting current status of procedure from master... 2023-07-22T12:38:00,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-07-22T12:38:00,796 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:39283=1690029457333} newestTimestamps: {jenkins-hbase3.apache.org:39283=1690029469041} 2023-07-22T12:38:00,799 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029417237.meta 2023-07-22T12:38:00,799 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029480628.meta 2023-07-22T12:38:00,799 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029469041 2023-07-22T12:38:00,800 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029480602 2023-07-22T12:38:00,800 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029438065.meta 2023-07-22T12:38:00,800 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029457350.meta 2023-07-22T12:38:00,800 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029469061.meta 2023-07-22T12:38:01,212 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(146): Attempting to copy table info for:test-1690029423533 target: hdfs://localhost:33313/backupUT/backup_1690029476431/default/test-1690029423533 descriptor: 'test-1690029423533', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:38:01,212 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-07-22T12:38:01,215 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(151): Starting to write region info for table test-1690029423533 2023-07-22T12:38:01,620 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(158): Finished writing region info for table test-1690029423533 2023-07-22T12:38:02,031 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(146): Attempting to copy table info for:ns2:test-16900294235331 target: hdfs://localhost:33313/backupUT/backup_1690029476431/ns2/test-16900294235331 descriptor: 'ns2:test-16900294235331', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:38:02,031 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-07-22T12:38:02,034 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(151): Starting to write region info for table ns2:test-16900294235331 2023-07-22T12:38:02,440 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(158): Finished writing region info for table ns2:test-16900294235331 2023-07-22T12:38:02,446 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:33313/backupUT/.tmp/backup_1690029476431 from hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029469041 2023-07-22T12:38:02,447 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x214e2948 to 127.0.0.1:51445 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:38:02,452 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fba8d8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:38:02,453 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:38:02,454 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:55512, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:38:02,456 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-07-22T12:38:02,456 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1690029423533 2023-07-22T12:38:02,459 DEBUG [hconnection-0xbaa78ee-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:38:02,460 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:58316, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:38:02,461 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1690029423533: test-1690029423533; 2023-07-22T12:38:02,461 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16900294235331 2023-07-22T12:38:02,463 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16900294235331: ns2:test-16900294235331; 2023-07-22T12:38:02,463 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(694): Configuring 2 reduce partitions to match current region count for all tables 2023-07-22T12:38:02,463 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/hbase-staging/partitions_38229db5-9aa8-40b5-b722-b3d3ba02de87 2023-07-22T12:38:02,870 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:02,871 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:02,871 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:02,872 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:03,854 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/hadoop-3342835939182815388.jar 2023-07-22T12:38:03,854 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:03,855 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:03,855 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:03,856 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:03,856 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:03,856 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:03,857 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:03,857 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:03,857 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:03,858 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:38:03,859 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:38:03,859 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:38:03,859 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:38:03,860 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:38:03,860 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:38:03,861 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:38:03,861 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:38:03,861 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:38:03,862 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:38:03,862 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:38:03,862 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:38:03,863 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:03,863 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:03,864 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:03,864 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:03,864 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:03,865 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:03,865 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:38:03,865 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: default:test-1690029423533,ns2:test-16900294235331 2023-07-22T12:38:03,865 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:38:03,866 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x214e2948 to 127.0.0.1:51445 2023-07-22T12:38:03,866 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:38:03,867 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:38:04,842 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/hadoop-4189634423933634974.jar 2023-07-22T12:38:04,848 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-07-22T12:38:04,867 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029469041 for WAL files 2023-07-22T12:38:04,868 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029469041 2023-07-22T12:38:04,952 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/mapreduce.cluster.local.dir/1690029484912/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:38:04,952 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/mapreduce.cluster.local.dir/1690029484912/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:38:05,022 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029469041 for hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029469041 (-9223372036854775808:9223372036854775807) length:13396 2023-07-22T12:38:05,029 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-07-22T12:38:05,029 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-07-22T12:38:05,069 DEBUG [pool-228-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x09f4b678 to 127.0.0.1:51445 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:38:05,074 DEBUG [pool-228-thread-1 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19f6f101, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:38:05,076 DEBUG [hconnection-0xc4d590c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:38:05,078 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:58324, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:38:05,080 DEBUG [pool-228-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x09f4b678 to 127.0.0.1:51445 2023-07-22T12:38:05,080 DEBUG [pool-228-thread-1 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:38:05,081 DEBUG [pool-228-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-t1100] 2023-07-22T12:38:05,081 DEBUG [pool-228-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-07-22T12:38:05,091 DEBUG [pool-228-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33313/backupUT/.tmp/backup_1690029476431/_temporary/0/_temporary/attempt_local1842503276_0007_r_000001_0/default/test-1690029423533/f/9b9424468fd24808a1bf1fb210f6a091 is 42, key is row-t1100/f:q1/1690029476358/Put/seqid=0 2023-07-22T12:38:05,092 WARN [Thread-1156 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:39283] Specified favored nodes: [jenkins-hbase3.apache.org:39283] 2023-07-22T12:38:05,999 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:33313/backupUT 2023-07-22T12:38:06,000 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1690029476431 2023-07-22T12:38:06,000 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-07-22T12:38:06,005 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:33313/backupUT/.tmp/backup_1690029476431, hdfs://localhost:33313/backupUT] 2023-07-22T12:38:06,028 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-07-22T12:38:06,607 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-07-22T12:38:06,609 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1690029476431 2023-07-22T12:38:06,609 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 9086 bytes copied." 2023-07-22T12:38:06,609 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local727958580_0008 completed: true true 2023-07-22T12:38:06,612 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=5019980 FILE: Number of bytes written=9838647 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=207029 HDFS: Number of bytes written=2395874 HDFS: Number of read operations=740 HDFS: Number of large read operations=0 HDFS: Number of write operations=302 Map-Reduce Framework Map input records=6 Map output records=0 Input split bytes=341 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1355808768 File Input Format Counters Bytes Read=1478 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=9086 Bytes Expected=9086 Files Copied=2 DIR_COPY=4 2023-07-22T12:38:06,612 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:33313/backupUT/.tmp/backup_1690029476431 to hdfs://localhost:33313/backupUT finished. 2023-07-22T12:38:06,621 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(433): deleted 0 original bulkload rows 2023-07-22T12:38:06,622 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029476431 2023-07-22T12:38:06,625 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029464859 to construct the dependency. 2023-07-22T12:38:06,625 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029464859 2023-07-22T12:38:06,627 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029464859/.backup.manifest 2023-07-22T12:38:06,627 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029464859BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:06,627 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029453150 to construct the dependency. 2023-07-22T12:38:06,628 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029453150 2023-07-22T12:38:06,629 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029453150/.backup.manifest 2023-07-22T12:38:06,630 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029453150BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:06,630 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(340): Got 3 ancestors for the current backup. 2023-07-22T12:38:06,630 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029476431 2023-07-22T12:38:06,631 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029464859 to construct the dependency. 2023-07-22T12:38:06,631 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029464859 2023-07-22T12:38:06,634 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029464859/.backup.manifest 2023-07-22T12:38:06,634 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029464859BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:06,634 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029453150 to construct the dependency. 2023-07-22T12:38:06,634 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029453150 2023-07-22T12:38:06,636 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029453150/.backup.manifest 2023-07-22T12:38:06,636 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029453150BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:06,636 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(340): Got 3 ancestors for the current backup. 2023-07-22T12:38:07,044 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33313/backupUT/backup_1690029476431/.backup.manifest 2023-07-22T12:38:07,044 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029476431 2023-07-22T12:38:07,047 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029464859 to construct the dependency. 2023-07-22T12:38:07,047 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029464859 2023-07-22T12:38:07,050 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029464859/.backup.manifest 2023-07-22T12:38:07,050 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029464859BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:07,050 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029453150 to construct the dependency. 2023-07-22T12:38:07,050 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029453150 2023-07-22T12:38:07,052 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029453150/.backup.manifest 2023-07-22T12:38:07,052 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029453150BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:07,052 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(340): Got 3 ancestors for the current backup. 2023-07-22T12:38:07,052 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029476431 2023-07-22T12:38:07,054 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029464859 to construct the dependency. 2023-07-22T12:38:07,054 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029464859 2023-07-22T12:38:07,056 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029464859/.backup.manifest 2023-07-22T12:38:07,056 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029464859BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:07,056 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029453150 to construct the dependency. 2023-07-22T12:38:07,056 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029453150 2023-07-22T12:38:07,058 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029453150/.backup.manifest 2023-07-22T12:38:07,059 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029453150BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:07,059 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(340): Got 3 ancestors for the current backup. 2023-07-22T12:38:07,465 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33313/backupUT/backup_1690029476431/.backup.manifest 2023-07-22T12:38:07,465 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029476431 2023-07-22T12:38:07,467 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029464859 to construct the dependency. 2023-07-22T12:38:07,467 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029464859 2023-07-22T12:38:07,470 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029464859/.backup.manifest 2023-07-22T12:38:07,470 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029464859BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:07,470 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029453150 to construct the dependency. 2023-07-22T12:38:07,470 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029453150 2023-07-22T12:38:07,472 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029453150/.backup.manifest 2023-07-22T12:38:07,472 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029453150BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:07,472 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(340): Got 3 ancestors for the current backup. 2023-07-22T12:38:07,878 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33313/backupUT/backup_1690029476431/.backup.manifest 2023-07-22T12:38:07,878 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(358): Backup backup_1690029476431 finished: type=INCREMENTAL,tablelist=test-1690029423533;ns2:test-16900294235331,targetRootDir=hdfs://localhost:33313/backupUT,startts=1690029480577,completets=1690029486621,bytescopied=0 2023-07-22T12:38:07,878 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:38:07,883 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:38:07,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:38:07,886 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:38:07,887 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:38:07,890 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(376): Backup backup_1690029476431 completed. 2023-07-22T12:38:08,001 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:38:08,004 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:38:08,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029488004 (current time:1690029488004). 2023-07-22T12:38:08,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:38:08,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:38:08,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7c6ea63f to 127.0.0.1:51445 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:38:08,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c84986e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:38:08,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:38:08,013 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:58334, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:38:08,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7c6ea63f to 127.0.0.1:51445 2023-07-22T12:38:08,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:38:08,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:38:08,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure2.ProcedureExecutor(1033): Stored pid=44, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:38:08,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 44 2023-07-22T12:38:08,016 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:38:08,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=44 2023-07-22T12:38:08,017 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:38:08,018 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:38:08,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=44 2023-07-22T12:38:08,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=44 2023-07-22T12:38:08,424 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:38:08,424 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c}] 2023-07-22T12:38:08,426 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=45, ppid=44, state=RUNNABLE; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c 2023-07-22T12:38:08,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39283 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=45 2023-07-22T12:38:08,578 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. 2023-07-22T12:38:08,578 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HRegion(2819): Flushing b8af53427af1873ee686b4b22616f51c 2/2 column families, dataSize=1.37 KB heapSize=2.52 KB 2023-07-22T12:38:08,582 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/meta/8f72b04c78fb4f5f8c4135a5820932d6 is 171, key is trslm:hdfs://localhost:33313/backupUT\x00ns2:test-16900294235331/meta:log-roll-map/1690029486616/Put/seqid=0 2023-07-22T12:38:08,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=44 2023-07-22T12:38:08,987 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=531 B at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/meta/8f72b04c78fb4f5f8c4135a5820932d6 2023-07-22T12:38:08,992 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8f72b04c78fb4f5f8c4135a5820932d6 2023-07-22T12:38:08,995 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/session/a30af91d0ecb4b06a2e7606fcf805b33 is 360, key is session:backup_1690029476431/session:context/1690029487886/Put/seqid=0 2023-07-22T12:38:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=44 2023-07-22T12:38:09,399 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=873 B at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/session/a30af91d0ecb4b06a2e7606fcf805b33 2023-07-22T12:38:09,405 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a30af91d0ecb4b06a2e7606fcf805b33 2023-07-22T12:38:09,405 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/meta/8f72b04c78fb4f5f8c4135a5820932d6 as hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/8f72b04c78fb4f5f8c4135a5820932d6 2023-07-22T12:38:09,410 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/8f72b04c78fb4f5f8c4135a5820932d6, entries=4, sequenceid=50, filesize=5.7 K 2023-07-22T12:38:09,411 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/session/a30af91d0ecb4b06a2e7606fcf805b33 as hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/a30af91d0ecb4b06a2e7606fcf805b33 2023-07-22T12:38:09,416 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/a30af91d0ecb4b06a2e7606fcf805b33, entries=2, sequenceid=50, filesize=5.3 K 2023-07-22T12:38:09,417 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HRegion(3022): Finished flush of dataSize ~1.37 KB/1404, heapSize ~2.49 KB/2552, currentSize=0 B/0 for b8af53427af1873ee686b4b22616f51c in 839ms, sequenceid=50, compaction requested=true 2023-07-22T12:38:09,417 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HRegion(2520): Flush status journal for b8af53427af1873ee686b4b22616f51c: 2023-07-22T12:38:09,417 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. for snapshot_backup_system completed. 2023-07-22T12:38:09,417 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:38:09,417 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:38:09,417 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/8b4e5904fd5541caabafed2e0a7bbecc, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/d7524c21e0884ef7a726165d97bdf81e, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/4b24f1898bb44138bbfed9c444ed2091, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/8f72b04c78fb4f5f8c4135a5820932d6] hfiles 2023-07-22T12:38:09,417 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/8b4e5904fd5541caabafed2e0a7bbecc for snapshot=snapshot_backup_system 2023-07-22T12:38:09,417 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/d7524c21e0884ef7a726165d97bdf81e for snapshot=snapshot_backup_system 2023-07-22T12:38:09,417 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/4b24f1898bb44138bbfed9c444ed2091 for snapshot=snapshot_backup_system 2023-07-22T12:38:09,417 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/8f72b04c78fb4f5f8c4135a5820932d6 for snapshot=snapshot_backup_system 2023-07-22T12:38:09,417 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/e7980d1347624a0e8cf8cbac22176c14, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/40fe5705ee804503bdd7677bd59e3be8, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/882f60fdcf9248459ac9b39b6e8e1f01, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/43bbdf59d84f4c98ac7bd38ac51d7e69, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/a30af91d0ecb4b06a2e7606fcf805b33] hfiles 2023-07-22T12:38:09,417 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/e7980d1347624a0e8cf8cbac22176c14 for snapshot=snapshot_backup_system 2023-07-22T12:38:09,418 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/40fe5705ee804503bdd7677bd59e3be8 for snapshot=snapshot_backup_system 2023-07-22T12:38:09,418 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/882f60fdcf9248459ac9b39b6e8e1f01 for snapshot=snapshot_backup_system 2023-07-22T12:38:09,418 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/43bbdf59d84f4c98ac7bd38ac51d7e69 for snapshot=snapshot_backup_system 2023-07-22T12:38:09,418 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/a30af91d0ecb4b06a2e7606fcf805b33 for snapshot=snapshot_backup_system 2023-07-22T12:38:09,823 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. 2023-07-22T12:38:09,823 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=45 2023-07-22T12:38:09,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.HMaster(3994): Remote procedure done, pid=45 2023-07-22T12:38:09,823 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region b8af53427af1873ee686b4b22616f51c 2023-07-22T12:38:09,825 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=45, ppid=44, state=RUNNABLE; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c 2023-07-22T12:38:09,827 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=45, resume processing ppid=44 2023-07-22T12:38:09,827 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:38:09,827 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=45, ppid=44, state=SUCCESS; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c in 1.4010 sec 2023-07-22T12:38:09,828 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:38:09,828 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:38:09,828 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:38:09,829 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:38:10,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=44 2023-07-22T12:38:10,240 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:38:10,249 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:38:10,250 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:38:11,067 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:38:11,067 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 44 2023-07-22T12:38:11,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=44, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0530 sec 2023-07-22T12:38:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=44 2023-07-22T12:38:12,123 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 44 completed 2023-07-22T12:38:12,123 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(115): Backup backup_1690029487980 started at 1690029492123. 2023-07-22T12:38:12,125 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(119): Backup session backup_1690029487980 has been started. 2023-07-22T12:38:12,126 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalTableBackupClient(268): For incremental backup, current table set is [test-1690029423533, ns2:test-16900294235331] 2023-07-22T12:38:12,132 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(76): StartCode 1690029469041for backupID backup_1690029487980 2023-07-22T12:38:12,132 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-07-22T12:38:12,132 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-07-22T12:38:12,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-07-22T12:38:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-07-22T12:38:12,133 INFO [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-07-22T12:38:12,133 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-07-22T12:38:12,134 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-07-22T12:38:12,134 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-07-22T12:38:12,134 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-07-22T12:38:12,134 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:38:12,135 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-07-22T12:38:12,135 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:12,135 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:38:12,136 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:38:12,136 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:12,136 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-07-22T12:38:12,136 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-07-22T12:38:12,136 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-07-22T12:38:12,137 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-07-22T12:38:12,137 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:38:12,137 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-07-22T12:38:12,137 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-07-22T12:38:12,137 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-07-22T12:38:12,137 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-07-22T12:38:12,137 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-07-22T12:38:12,137 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-07-22T12:38:12,138 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-07-22T12:38:12,138 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-07-22T12:38:12,138 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,39283,1690029414077' joining acquired barrier for procedure (rolllog) in zk 2023-07-22T12:38:12,140 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:12,140 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-07-22T12:38:12,140 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:12,140 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:38:12,140 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:38:12,140 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-07-22T12:38:12,140 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-07-22T12:38:12,140 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:38:12,141 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:38:12,141 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:38:12,141 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:12,142 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:38:12,142 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,39283,1690029414077' joining acquired barrier for procedure 'rolllog' on coordinator 2023-07-22T12:38:12,142 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@205a164b[Count = 0] remaining members to acquire global barrier 2023-07-22T12:38:12,142 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-07-22T12:38:12,142 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-07-22T12:38:12,143 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:38:12,143 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-07-22T12:38:12,143 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-07-22T12:38:12,143 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-07-22T12:38:12,144 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:12,144 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-07-22T12:38:12,144 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-07-22T12:38:12,144 DEBUG [rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:12,144 INFO [rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1690029480602 highest: 1690029480602 on jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:12,145 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C39283%2C1690029414077:(num 1690029480602) roll requested 2023-07-22T12:38:12,161 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029492145, exclude list is [], retry=0 2023-07-22T12:38:12,165 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK] 2023-07-22T12:38:12,167 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029480602 with entries=109, filesize=13.56 KB; new WAL /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029492145 2023-07-22T12:38:12,167 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK]] 2023-07-22T12:38:12,167 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029480602 is not closed yet, will try archiving it next time 2023-07-22T12:38:12,167 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta:.meta(num 1690029480628) roll requested 2023-07-22T12:38:12,169 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029480602 not finished, retry = 0 2023-07-22T12:38:12,184 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029492167.meta, exclude list is [], retry=0 2023-07-22T12:38:12,187 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK] 2023-07-22T12:38:12,188 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029480628.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029492167.meta 2023-07-22T12:38:12,189 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK]] 2023-07-22T12:38:12,189 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029480628.meta is not closed yet, will try archiving it next time 2023-07-22T12:38:12,190 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029480628.meta not finished, retry = 0 2023-07-22T12:38:12,205 DEBUG [rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2023-07-22T12:38:12,205 INFO [rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1690029492145 on jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:12,214 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-07-22T12:38:12,214 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,39283,1690029414077' in zk 2023-07-22T12:38:12,220 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-07-22T12:38:12,220 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:12,220 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:38:12,220 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-07-22T12:38:12,221 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:12,221 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:38:12,221 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:38:12,222 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:38:12,222 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:38:12,222 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:38:12,222 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:12,223 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:38:12,223 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:38:12,223 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:12,224 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,39283,1690029414077': 2023-07-22T12:38:12,224 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,39283,1690029414077' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-07-22T12:38:12,224 INFO [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-07-22T12:38:12,224 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-07-22T12:38:12,224 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-07-22T12:38:12,224 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-07-22T12:38:12,224 INFO [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-07-22T12:38:12,226 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:38:12,226 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:38:12,226 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-07-22T12:38:12,226 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-07-22T12:38:12,226 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-07-22T12:38:12,226 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:38:12,226 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:38:12,226 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:38:12,226 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:38:12,226 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:38:12,226 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:38:12,226 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:12,227 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:38:12,227 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:38:12,227 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:38:12,227 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:38:12,228 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:12,228 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:12,228 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:38:12,229 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:38:12,229 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:12,234 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-07-22T12:38:12,234 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:12,234 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-07-22T12:38:12,234 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:38:12,234 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-07-22T12:38:12,235 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:38:12,235 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:12,235 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-07-22T12:38:12,235 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:38:12,235 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:38:12,235 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:38:12,235 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:38:12,235 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-07-22T12:38:12,236 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-07-22T12:38:12,236 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2023-07-22T12:38:12,236 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin(2751): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-07-22T12:38:12,236 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin(2759): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-07-22T12:38:12,236 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:38:12,236 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:38:12,291 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029480628.meta to hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029480628.meta 2023-07-22T12:38:12,336 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin(2765): Getting current status of procedure from master... 2023-07-22T12:38:12,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-07-22T12:38:12,340 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:39283=1690029469041} newestTimestamps: {jenkins-hbase3.apache.org:39283=1690029480602} 2023-07-22T12:38:12,341 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029417237.meta 2023-07-22T12:38:12,342 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029492167.meta 2023-07-22T12:38:12,342 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029480602 2023-07-22T12:38:12,342 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029492145 2023-07-22T12:38:12,342 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029438065.meta 2023-07-22T12:38:12,342 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029457350.meta 2023-07-22T12:38:12,343 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029469061.meta 2023-07-22T12:38:12,343 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029480628.meta 2023-07-22T12:38:12,752 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(146): Attempting to copy table info for:test-1690029423533 target: hdfs://localhost:33313/backupUT/backup_1690029487980/default/test-1690029423533 descriptor: 'test-1690029423533', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:38:12,752 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-07-22T12:38:12,754 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(151): Starting to write region info for table test-1690029423533 2023-07-22T12:38:13,160 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(158): Finished writing region info for table test-1690029423533 2023-07-22T12:38:13,569 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(146): Attempting to copy table info for:ns2:test-16900294235331 target: hdfs://localhost:33313/backupUT/backup_1690029487980/ns2/test-16900294235331 descriptor: 'ns2:test-16900294235331', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:38:13,569 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-07-22T12:38:13,571 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(151): Starting to write region info for table ns2:test-16900294235331 2023-07-22T12:38:13,977 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(158): Finished writing region info for table ns2:test-16900294235331 2023-07-22T12:38:13,982 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:33313/backupUT/.tmp/backup_1690029487980 from hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029480602 2023-07-22T12:38:13,982 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x15c4d745 to 127.0.0.1:51445 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:38:13,992 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13d703e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:38:13,993 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:38:13,994 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:37390, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:38:13,996 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-07-22T12:38:13,996 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1690029423533 2023-07-22T12:38:14,000 DEBUG [hconnection-0x1f96f642-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:38:14,001 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:50138, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:38:14,003 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1690029423533: test-1690029423533; 2023-07-22T12:38:14,003 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16900294235331 2023-07-22T12:38:14,007 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16900294235331: ns2:test-16900294235331; 2023-07-22T12:38:14,007 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(694): Configuring 2 reduce partitions to match current region count for all tables 2023-07-22T12:38:14,007 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/hbase-staging/partitions_b838bb38-6b4e-4a05-9540-022436c2681e 2023-07-22T12:38:14,415 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:14,415 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:14,416 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:14,416 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:15,382 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/hadoop-3089475444559053527.jar 2023-07-22T12:38:15,383 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:15,383 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:15,383 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:15,384 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:15,384 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:15,385 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:15,385 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:15,385 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:15,386 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:15,386 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:38:15,387 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:38:15,387 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:38:15,388 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:38:15,388 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:38:15,388 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:38:15,389 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:38:15,389 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:38:15,389 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:38:15,390 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:38:15,390 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:38:15,391 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:38:15,391 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:15,391 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:15,392 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:15,392 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:15,392 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:15,393 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:15,393 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:38:15,393 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: default:test-1690029423533,ns2:test-16900294235331 2023-07-22T12:38:15,393 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:38:15,394 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x15c4d745 to 127.0.0.1:51445 2023-07-22T12:38:15,394 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:38:15,395 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:38:16,357 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/hadoop-307201259420853688.jar 2023-07-22T12:38:16,363 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-07-22T12:38:16,377 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029480602 for WAL files 2023-07-22T12:38:16,379 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029480602 2023-07-22T12:38:16,448 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/mapreduce.cluster.local.dir/1690029496415/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:38:16,448 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/mapreduce.cluster.local.dir/1690029496415/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:38:16,532 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029480602 for hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029480602 (-9223372036854775808:9223372036854775807) length:13891 2023-07-22T12:38:16,538 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-07-22T12:38:16,538 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-07-22T12:38:16,553 DEBUG [pool-241-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4843a9e1 to 127.0.0.1:51445 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:38:16,560 DEBUG [pool-241-thread-1 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@562ae118, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:38:16,562 DEBUG [hconnection-0x4ffccfff-metaLookup-shared--pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:38:16,563 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:50150, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:38:16,565 DEBUG [pool-241-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4843a9e1 to 127.0.0.1:51445 2023-07-22T12:38:16,565 DEBUG [pool-241-thread-1 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:38:16,567 DEBUG [pool-241-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-t1100] 2023-07-22T12:38:16,567 DEBUG [pool-241-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-07-22T12:38:16,575 DEBUG [pool-241-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33313/backupUT/.tmp/backup_1690029487980/_temporary/0/_temporary/attempt_local1892860633_0009_r_000000_0/ns2/test-16900294235331/f/c8954c82b30a474d80fd78cafa3d8524 is 42, key is row-t1100/f:q1/1690029487902/Put/seqid=0 2023-07-22T12:38:16,577 WARN [Thread-1289 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:39283] Specified favored nodes: [jenkins-hbase3.apache.org:39283] 2023-07-22T12:38:17,482 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:33313/backupUT 2023-07-22T12:38:17,482 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1690029487980 2023-07-22T12:38:17,482 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-07-22T12:38:17,487 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:33313/backupUT/.tmp/backup_1690029487980, hdfs://localhost:33313/backupUT] 2023-07-22T12:38:17,513 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-07-22T12:38:18,092 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-07-22T12:38:18,094 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1690029487980 2023-07-22T12:38:18,094 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 9405 bytes copied." 2023-07-22T12:38:18,094 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local109195927_0010 completed: true true 2023-07-22T12:38:18,098 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=5650448 FILE: Number of bytes written=12237640 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=261731 HDFS: Number of bytes written=2437038 HDFS: Number of read operations=892 HDFS: Number of large read operations=0 HDFS: Number of write operations=349 Map-Reduce Framework Map input records=6 Map output records=0 Input split bytes=341 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1365245952 File Input Format Counters Bytes Read=1452 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=9405 Bytes Expected=9405 Files Copied=2 DIR_COPY=4 2023-07-22T12:38:18,098 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:33313/backupUT/.tmp/backup_1690029487980 to hdfs://localhost:33313/backupUT finished. 2023-07-22T12:38:18,112 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(433): deleted 0 original bulkload rows 2023-07-22T12:38:18,112 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029487980 2023-07-22T12:38:18,116 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029476431 to construct the dependency. 2023-07-22T12:38:18,117 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029476431 2023-07-22T12:38:18,119 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029476431/.backup.manifest 2023-07-22T12:38:18,119 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029476431BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:18,119 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029464859 to construct the dependency. 2023-07-22T12:38:18,119 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029464859 2023-07-22T12:38:18,121 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029464859/.backup.manifest 2023-07-22T12:38:18,122 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029464859BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:18,122 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029453150 to construct the dependency. 2023-07-22T12:38:18,122 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029453150 2023-07-22T12:38:18,123 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029453150/.backup.manifest 2023-07-22T12:38:18,123 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029453150BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:18,124 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(340): Got 4 ancestors for the current backup. 2023-07-22T12:38:18,124 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029487980 2023-07-22T12:38:18,126 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029476431 to construct the dependency. 2023-07-22T12:38:18,126 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029476431 2023-07-22T12:38:18,128 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029476431/.backup.manifest 2023-07-22T12:38:18,128 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029476431BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:18,128 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029464859 to construct the dependency. 2023-07-22T12:38:18,128 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029464859 2023-07-22T12:38:18,130 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029464859/.backup.manifest 2023-07-22T12:38:18,130 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029464859BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:18,130 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029453150 to construct the dependency. 2023-07-22T12:38:18,130 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029453150 2023-07-22T12:38:18,132 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029453150/.backup.manifest 2023-07-22T12:38:18,132 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029453150BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:18,132 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(340): Got 4 ancestors for the current backup. 2023-07-22T12:38:18,538 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33313/backupUT/backup_1690029487980/.backup.manifest 2023-07-22T12:38:18,539 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029487980 2023-07-22T12:38:18,541 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029476431 to construct the dependency. 2023-07-22T12:38:18,541 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029476431 2023-07-22T12:38:18,545 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029476431/.backup.manifest 2023-07-22T12:38:18,545 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029476431BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:18,545 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029464859 to construct the dependency. 2023-07-22T12:38:18,545 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029464859 2023-07-22T12:38:18,547 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029464859/.backup.manifest 2023-07-22T12:38:18,547 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029464859BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:18,547 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029453150 to construct the dependency. 2023-07-22T12:38:18,547 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029453150 2023-07-22T12:38:18,550 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029453150/.backup.manifest 2023-07-22T12:38:18,550 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029453150BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:18,550 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(340): Got 4 ancestors for the current backup. 2023-07-22T12:38:18,550 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029487980 2023-07-22T12:38:18,552 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029476431 to construct the dependency. 2023-07-22T12:38:18,553 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029476431 2023-07-22T12:38:18,555 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029476431/.backup.manifest 2023-07-22T12:38:18,555 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029476431BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:18,555 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029464859 to construct the dependency. 2023-07-22T12:38:18,555 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029464859 2023-07-22T12:38:18,557 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029464859/.backup.manifest 2023-07-22T12:38:18,558 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029464859BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:18,558 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029453150 to construct the dependency. 2023-07-22T12:38:18,558 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029453150 2023-07-22T12:38:18,560 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029453150/.backup.manifest 2023-07-22T12:38:18,560 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029453150BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:18,560 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(340): Got 4 ancestors for the current backup. 2023-07-22T12:38:18,966 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33313/backupUT/backup_1690029487980/.backup.manifest 2023-07-22T12:38:18,966 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029487980 2023-07-22T12:38:18,968 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029476431 to construct the dependency. 2023-07-22T12:38:18,969 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029476431 2023-07-22T12:38:18,971 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029476431/.backup.manifest 2023-07-22T12:38:18,971 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029476431BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:18,971 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029464859 to construct the dependency. 2023-07-22T12:38:18,971 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029464859 2023-07-22T12:38:18,973 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029464859/.backup.manifest 2023-07-22T12:38:18,973 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029464859BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:18,973 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029453150 to construct the dependency. 2023-07-22T12:38:18,974 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029453150 2023-07-22T12:38:18,976 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029453150/.backup.manifest 2023-07-22T12:38:18,976 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029453150BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:18,976 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(340): Got 4 ancestors for the current backup. 2023-07-22T12:38:19,381 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33313/backupUT/backup_1690029487980/.backup.manifest 2023-07-22T12:38:19,381 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(358): Backup backup_1690029487980 finished: type=INCREMENTAL,tablelist=test-1690029423533;ns2:test-16900294235331,targetRootDir=hdfs://localhost:33313/backupUT,startts=1690029492123,completets=1690029498112,bytescopied=0 2023-07-22T12:38:19,381 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:38:19,387 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:38:19,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:38:19,390 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:38:19,392 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:38:19,394 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(376): Backup backup_1690029487980 completed. 2023-07-22T12:38:19,414 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:38:19,418 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:38:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029499418 (current time:1690029499418). 2023-07-22T12:38:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:38:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:38:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6a3f6fda to 127.0.0.1:51445 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:38:19,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55386a37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:38:19,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:38:19,426 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:50162, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:38:19,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6a3f6fda to 127.0.0.1:51445 2023-07-22T12:38:19,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:38:19,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:38:19,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure2.ProcedureExecutor(1033): Stored pid=46, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:38:19,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 46 2023-07-22T12:38:19,429 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:38:19,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-07-22T12:38:19,430 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:38:19,432 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:38:19,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-07-22T12:38:19,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-07-22T12:38:19,838 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:38:19,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c}] 2023-07-22T12:38:19,840 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=47, ppid=46, state=RUNNABLE; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c 2023-07-22T12:38:19,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39283 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=47 2023-07-22T12:38:19,992 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. 2023-07-22T12:38:19,992 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.HRegion(2819): Flushing b8af53427af1873ee686b4b22616f51c 2/2 column families, dataSize=1.37 KB heapSize=2.52 KB 2023-07-22T12:38:19,996 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/meta/f1b01e23079e4b62937f7c002d957751 is 171, key is trslm:hdfs://localhost:33313/backupUT\x00ns2:test-16900294235331/meta:log-roll-map/1690029498104/Put/seqid=0 2023-07-22T12:38:20,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-07-22T12:38:20,401 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=531 B at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/meta/f1b01e23079e4b62937f7c002d957751 2023-07-22T12:38:20,407 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: f1b01e23079e4b62937f7c002d957751 2023-07-22T12:38:20,411 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/session/e19bff70c6164f2581f82c4faaec10d1 is 360, key is session:backup_1690029487980/session:context/1690029499391/Put/seqid=0 2023-07-22T12:38:20,416 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=873 B at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/session/e19bff70c6164f2581f82c4faaec10d1 2023-07-22T12:38:20,421 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e19bff70c6164f2581f82c4faaec10d1 2023-07-22T12:38:20,422 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/meta/f1b01e23079e4b62937f7c002d957751 as hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/f1b01e23079e4b62937f7c002d957751 2023-07-22T12:38:20,428 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/f1b01e23079e4b62937f7c002d957751, entries=4, sequenceid=61, filesize=5.7 K 2023-07-22T12:38:20,428 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/session/e19bff70c6164f2581f82c4faaec10d1 as hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/e19bff70c6164f2581f82c4faaec10d1 2023-07-22T12:38:20,434 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/e19bff70c6164f2581f82c4faaec10d1, entries=2, sequenceid=61, filesize=5.3 K 2023-07-22T12:38:20,435 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.HRegion(3022): Finished flush of dataSize ~1.37 KB/1404, heapSize ~2.49 KB/2552, currentSize=0 B/0 for b8af53427af1873ee686b4b22616f51c in 443ms, sequenceid=61, compaction requested=true 2023-07-22T12:38:20,435 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.HRegion(2520): Flush status journal for b8af53427af1873ee686b4b22616f51c: 2023-07-22T12:38:20,435 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. for snapshot_backup_system completed. 2023-07-22T12:38:20,435 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:38:20,435 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:38:20,435 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/8b4e5904fd5541caabafed2e0a7bbecc, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/d7524c21e0884ef7a726165d97bdf81e, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/4b24f1898bb44138bbfed9c444ed2091, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/8f72b04c78fb4f5f8c4135a5820932d6, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/f1b01e23079e4b62937f7c002d957751] hfiles 2023-07-22T12:38:20,435 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/8b4e5904fd5541caabafed2e0a7bbecc for snapshot=snapshot_backup_system 2023-07-22T12:38:20,435 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/d7524c21e0884ef7a726165d97bdf81e for snapshot=snapshot_backup_system 2023-07-22T12:38:20,435 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/4b24f1898bb44138bbfed9c444ed2091 for snapshot=snapshot_backup_system 2023-07-22T12:38:20,435 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/8f72b04c78fb4f5f8c4135a5820932d6 for snapshot=snapshot_backup_system 2023-07-22T12:38:20,435 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/f1b01e23079e4b62937f7c002d957751 for snapshot=snapshot_backup_system 2023-07-22T12:38:20,435 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/e7980d1347624a0e8cf8cbac22176c14, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/40fe5705ee804503bdd7677bd59e3be8, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/882f60fdcf9248459ac9b39b6e8e1f01, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/43bbdf59d84f4c98ac7bd38ac51d7e69, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/a30af91d0ecb4b06a2e7606fcf805b33, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/e19bff70c6164f2581f82c4faaec10d1] hfiles 2023-07-22T12:38:20,435 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (1/6): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/e7980d1347624a0e8cf8cbac22176c14 for snapshot=snapshot_backup_system 2023-07-22T12:38:20,435 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (2/6): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/40fe5705ee804503bdd7677bd59e3be8 for snapshot=snapshot_backup_system 2023-07-22T12:38:20,435 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (3/6): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/882f60fdcf9248459ac9b39b6e8e1f01 for snapshot=snapshot_backup_system 2023-07-22T12:38:20,435 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (4/6): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/43bbdf59d84f4c98ac7bd38ac51d7e69 for snapshot=snapshot_backup_system 2023-07-22T12:38:20,435 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (5/6): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/a30af91d0ecb4b06a2e7606fcf805b33 for snapshot=snapshot_backup_system 2023-07-22T12:38:20,435 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (6/6): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/e19bff70c6164f2581f82c4faaec10d1 for snapshot=snapshot_backup_system 2023-07-22T12:38:20,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-07-22T12:38:20,843 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. 2023-07-22T12:38:20,843 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=47 2023-07-22T12:38:20,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.HMaster(3994): Remote procedure done, pid=47 2023-07-22T12:38:20,843 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region b8af53427af1873ee686b4b22616f51c 2023-07-22T12:38:20,844 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=47, ppid=46, state=RUNNABLE; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c 2023-07-22T12:38:20,846 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=47, resume processing ppid=46 2023-07-22T12:38:20,847 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:38:20,847 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=47, ppid=46, state=SUCCESS; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c in 1.0070 sec 2023-07-22T12:38:20,847 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:38:20,848 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:38:20,848 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:38:20,849 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:38:21,258 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:38:21,268 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:38:21,268 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:38:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-07-22T12:38:22,086 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:38:22,086 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 46 2023-07-22T12:38:22,087 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=46, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.6590 sec 2023-07-22T12:38:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-07-22T12:38:23,536 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 46 completed 2023-07-22T12:38:23,537 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(115): Backup backup_1690029499404 started at 1690029503536. 2023-07-22T12:38:23,538 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(119): Backup session backup_1690029499404 has been started. 2023-07-22T12:38:23,542 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-07-22T12:38:23,543 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-07-22T12:38:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-07-22T12:38:23,543 INFO [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-07-22T12:38:23,544 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-07-22T12:38:23,545 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-07-22T12:38:23,545 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-07-22T12:38:23,545 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-07-22T12:38:23,545 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:38:23,552 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:23,552 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-07-22T12:38:23,552 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:38:23,552 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:38:23,552 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:23,552 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-07-22T12:38:23,552 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-07-22T12:38:23,553 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-07-22T12:38:23,553 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-07-22T12:38:23,553 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:38:23,553 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-07-22T12:38:23,553 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-07-22T12:38:23,553 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-07-22T12:38:23,553 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-07-22T12:38:23,553 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-07-22T12:38:23,554 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-07-22T12:38:23,554 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-07-22T12:38:23,554 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,39283,1690029414077' joining acquired barrier for procedure (rolllog) in zk 2023-07-22T12:38:23,556 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-07-22T12:38:23,556 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:23,557 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:23,557 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:38:23,557 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:38:23,557 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-07-22T12:38:23,557 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-07-22T12:38:23,557 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:38:23,557 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:38:23,558 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:38:23,558 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:23,558 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:38:23,559 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,39283,1690029414077' joining acquired barrier for procedure 'rolllog' on coordinator 2023-07-22T12:38:23,559 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@10b7be6f[Count = 0] remaining members to acquire global barrier 2023-07-22T12:38:23,559 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-07-22T12:38:23,559 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-07-22T12:38:23,560 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:38:23,560 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-07-22T12:38:23,560 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-07-22T12:38:23,560 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-07-22T12:38:23,560 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:23,560 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-07-22T12:38:23,560 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-07-22T12:38:23,560 DEBUG [rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:23,560 INFO [rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1690029492145 highest: 1690029492145 on jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:23,560 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C39283%2C1690029414077:(num 1690029492145) roll requested 2023-07-22T12:38:23,571 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029503560, exclude list is [], retry=0 2023-07-22T12:38:23,577 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK] 2023-07-22T12:38:23,579 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029492145 with entries=10, filesize=2.51 KB; new WAL /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029503560 2023-07-22T12:38:23,579 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK]] 2023-07-22T12:38:23,579 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029492145 is not closed yet, will try archiving it next time 2023-07-22T12:38:23,579 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta:.meta(num 1690029492167) roll requested 2023-07-22T12:38:23,581 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029492145 not finished, retry = 0 2023-07-22T12:38:23,590 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029503579.meta, exclude list is [], retry=0 2023-07-22T12:38:23,593 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK] 2023-07-22T12:38:23,595 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029492167.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029503579.meta 2023-07-22T12:38:23,595 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK]] 2023-07-22T12:38:23,595 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029492167.meta is not closed yet, will try archiving it next time 2023-07-22T12:38:23,597 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029492167.meta not finished, retry = 0 2023-07-22T12:38:23,601 DEBUG [rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-07-22T12:38:23,601 INFO [rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1690029503560 on jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:23,608 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-07-22T12:38:23,608 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,39283,1690029414077' in zk 2023-07-22T12:38:23,611 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:23,611 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-07-22T12:38:23,611 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:38:23,611 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-07-22T12:38:23,612 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:23,612 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:38:23,612 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:38:23,612 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:38:23,612 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:38:23,613 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:38:23,613 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:23,613 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:38:23,613 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:38:23,614 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:23,614 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,39283,1690029414077': 2023-07-22T12:38:23,614 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,39283,1690029414077' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-07-22T12:38:23,614 INFO [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-07-22T12:38:23,614 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-07-22T12:38:23,614 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-07-22T12:38:23,614 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-07-22T12:38:23,614 INFO [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-07-22T12:38:23,617 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:38:23,617 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:38:23,617 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-07-22T12:38:23,617 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-07-22T12:38:23,617 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-07-22T12:38:23,617 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:38:23,617 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:38:23,617 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:38:23,617 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:38:23,617 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:38:23,617 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:38:23,618 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:23,618 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:38:23,618 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:38:23,618 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:38:23,619 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:38:23,619 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:23,619 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:23,619 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:38:23,620 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:38:23,620 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:23,623 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:23,623 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-07-22T12:38:23,623 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-07-22T12:38:23,623 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:23,623 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-07-22T12:38:23,623 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:38:23,623 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:38:23,623 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:38:23,623 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-07-22T12:38:23,623 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:38:23,623 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-07-22T12:38:23,623 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:38:23,623 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-07-22T12:38:23,624 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:38:23,624 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2023-07-22T12:38:23,624 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin(2751): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-07-22T12:38:23,624 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:38:23,624 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:38:23,624 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin(2759): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-07-22T12:38:23,698 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029492167.meta to hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029492167.meta 2023-07-22T12:38:23,724 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin(2765): Getting current status of procedure from master... 2023-07-22T12:38:23,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-07-22T12:38:23,728 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1690029503728_ns3_test-16900294235332 table=ns3:test-16900294235332 type=FLUSH ttl=0 } 2023-07-22T12:38:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029503728 (current time:1690029503728). 2023-07-22T12:38:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:38:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1690029503728_ns3_test-16900294235332 VERSION not specified, setting to 2 2023-07-22T12:38:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5ff79665 to 127.0.0.1:51445 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:38:23,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ea763fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:38:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:38:23,739 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:37954, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:38:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5ff79665 to 127.0.0.1:51445 2023-07-22T12:38:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:38:23,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:38:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure2.ProcedureExecutor(1033): Stored pid=48, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1690029503728_ns3_test-16900294235332 table=ns3:test-16900294235332 type=FLUSH ttl=0 } 2023-07-22T12:38:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1690029503728_ns3_test-16900294235332 table=ns3:test-16900294235332 type=FLUSH ttl=0 }, snapshot procedure id = 48 2023-07-22T12:38:23,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=48 2023-07-22T12:38:23,742 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1690029503728_ns3_test-16900294235332 table=ns3:test-16900294235332 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:38:23,743 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1690029503728_ns3_test-16900294235332 table=ns3:test-16900294235332 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:38:23,745 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1690029503728_ns3_test-16900294235332 table=ns3:test-16900294235332 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:38:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=48 2023-07-22T12:38:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=48 2023-07-22T12:38:24,150 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1690029503728_ns3_test-16900294235332 table=ns3:test-16900294235332 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:38:24,151 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE; SnapshotRegionProcedure c29ae4e4c3e8a42ff3c19bb2bd28710f}] 2023-07-22T12:38:24,153 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=49, ppid=48, state=RUNNABLE; SnapshotRegionProcedure c29ae4e4c3e8a42ff3c19bb2bd28710f 2023-07-22T12:38:24,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39283 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=49 2023-07-22T12:38:24,304 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16900294235332,,1690029427482.c29ae4e4c3e8a42ff3c19bb2bd28710f. 2023-07-22T12:38:24,305 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegion(2520): Flush status journal for c29ae4e4c3e8a42ff3c19bb2bd28710f: 2023-07-22T12:38:24,306 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16900294235332,,1690029427482.c29ae4e4c3e8a42ff3c19bb2bd28710f. for snapshot_1690029503728_ns3_test-16900294235332 completed. 2023-07-22T12:38:24,306 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16900294235332,,1690029427482.c29ae4e4c3e8a42ff3c19bb2bd28710f.' region-info for snapshot=snapshot_1690029503728_ns3_test-16900294235332 2023-07-22T12:38:24,306 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:38:24,306 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-07-22T12:38:24,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=48 2023-07-22T12:38:24,712 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16900294235332,,1690029427482.c29ae4e4c3e8a42ff3c19bb2bd28710f. 2023-07-22T12:38:24,712 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=49 2023-07-22T12:38:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.HMaster(3994): Remote procedure done, pid=49 2023-07-22T12:38:24,712 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1690029503728_ns3_test-16900294235332 on region c29ae4e4c3e8a42ff3c19bb2bd28710f 2023-07-22T12:38:24,714 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=49, ppid=48, state=RUNNABLE; SnapshotRegionProcedure c29ae4e4c3e8a42ff3c19bb2bd28710f 2023-07-22T12:38:24,716 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=49, resume processing ppid=48 2023-07-22T12:38:24,716 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1690029503728_ns3_test-16900294235332 table=ns3:test-16900294235332 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:38:24,716 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=49, ppid=48, state=SUCCESS; SnapshotRegionProcedure c29ae4e4c3e8a42ff3c19bb2bd28710f in 563 msec 2023-07-22T12:38:24,716 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1690029503728_ns3_test-16900294235332 table=ns3:test-16900294235332 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:38:24,717 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1690029503728_ns3_test-16900294235332 table=ns3:test-16900294235332 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:38:24,717 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1690029503728_ns3_test-16900294235332 2023-07-22T12:38:24,718 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/.tmp/snapshot_1690029503728_ns3_test-16900294235332 2023-07-22T12:38:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=48 2023-07-22T12:38:25,127 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1690029503728_ns3_test-16900294235332 table=ns3:test-16900294235332 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:38:25,133 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1690029503728_ns3_test-16900294235332 table=ns3:test-16900294235332 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:38:25,133 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/.tmp/snapshot_1690029503728_ns3_test-16900294235332 to hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/snapshot_1690029503728_ns3_test-16900294235332 2023-07-22T12:38:25,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=48 2023-07-22T12:38:25,956 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1690029503728_ns3_test-16900294235332 table=ns3:test-16900294235332 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:38:25,956 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1690029503728_ns3_test-16900294235332 table=ns3:test-16900294235332 type=FLUSH ttl=0 }, snapshot procedure id = 48 2023-07-22T12:38:25,957 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=48, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1690029503728_ns3_test-16900294235332 table=ns3:test-16900294235332 type=FLUSH ttl=0 } in 2.2160 sec 2023-07-22T12:38:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=48 2023-07-22T12:38:27,849 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: ns3:test-16900294235332, procId: 48 completed 2023-07-22T12:38:27,849 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1690029499404 2023-07-22T12:38:27,849 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-07-22T12:38:27,849 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-07-22T12:38:27,849 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1690029499404_ns3:test-16900294235332 2023-07-22T12:38:27,849 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1690029503728_ns3_test-16900294235332 to hdfs://localhost:33313/backupUT/backup_1690029499404/ns3/test-16900294235332/ 2023-07-22T12:38:27,849 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-07-22T12:38:27,873 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:33313, inputRoot=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d 2023-07-22T12:38:27,873 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,33617,1690029410929_-1951653582_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:33313/backupUT/backup_1690029499404/ns3/test-16900294235332, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:33313/backupUT/backup_1690029499404/ns3/test-16900294235332/.hbase-snapshot/.tmp/snapshot_1690029503728_ns3_test-16900294235332 2023-07-22T12:38:27,873 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:33313, inputRoot=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d, snapshotDir=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/snapshot_1690029503728_ns3_test-16900294235332. 2023-07-22T12:38:27,882 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/snapshot_1690029503728_ns3_test-16900294235332 to hdfs://localhost:33313/backupUT/backup_1690029499404/ns3/test-16900294235332/.hbase-snapshot/.tmp/snapshot_1690029503728_ns3_test-16900294235332 2023-07-22T12:38:28,301 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:28,301 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:28,302 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:28,302 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:29,279 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/hadoop-3266788291592062995.jar 2023-07-22T12:38:29,280 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:29,280 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:29,281 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:29,281 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:29,281 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:29,282 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:29,282 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:29,282 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:29,283 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:29,283 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:38:29,284 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:38:29,284 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:38:29,285 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:38:29,285 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:38:29,286 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:38:29,286 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:38:29,286 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:38:29,287 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:38:29,287 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:38:29,287 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:38:29,288 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:38:29,288 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:38:29,289 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:38:29,289 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:38:29,290 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:38:29,290 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:38:29,291 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:38:29,292 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:38:29,325 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-07-22T12:38:29,348 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1690029503728_ns3_test-16900294235332' hfile list 2023-07-22T12:38:29,422 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/mapreduce.cluster.local.dir/1690029509387/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:38:29,423 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/mapreduce.cluster.local.dir/1690029509387/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:38:30,451 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-07-22T12:38:30,452 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-07-22T12:38:30,457 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1690029503728_ns3_test-16900294235332 2023-07-22T12:38:30,457 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1690029503728_ns3_test-16900294235332 finished. 2023-07-22T12:38:30,457 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(1014): ns3:test-16900294235332 2023-07-22T12:38:30,464 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029499404 2023-07-22T12:38:30,464 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-07-22T12:38:30,872 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33313/backupUT/backup_1690029499404/.backup.manifest 2023-07-22T12:38:30,872 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(358): Backup backup_1690029499404 finished: type=FULL,tablelist=ns3:test-16900294235332,targetRootDir=hdfs://localhost:33313/backupUT,startts=1690029503536,completets=1690029510464,bytescopied=0 2023-07-22T12:38:30,872 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-07-22T12:38:30,872 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1690029503728_ns3_test-16900294235332 2023-07-22T12:38:30,874 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1690029503728_ns3_test-16900294235332" 2023-07-22T12:38:30,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1690029503728_ns3_test-16900294235332 2023-07-22T12:38:30,877 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1690029503728_ns3_test-16900294235332 for backup backup_1690029499404 succeeded. 2023-07-22T12:38:30,878 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:38:30,883 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:38:30,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:38:30,885 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:38:30,887 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:38:30,893 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(376): Backup backup_1690029499404 completed. 2023-07-22T12:38:30,939 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:38:30,942 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:38:30,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029510942 (current time:1690029510942). 2023-07-22T12:38:30,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:38:30,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:38:30,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0f9bd3a4 to 127.0.0.1:51445 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:38:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f57c0d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:38:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:38:30,952 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:50280, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:38:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0f9bd3a4 to 127.0.0.1:51445 2023-07-22T12:38:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:38:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:38:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure2.ProcedureExecutor(1033): Stored pid=50, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:38:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 50 2023-07-22T12:38:30,956 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:38:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-07-22T12:38:30,956 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:38:30,958 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:38:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-07-22T12:38:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-07-22T12:38:31,364 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:38:31,364 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c}] 2023-07-22T12:38:31,366 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c 2023-07-22T12:38:31,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39283 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=51 2023-07-22T12:38:31,518 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. 2023-07-22T12:38:31,518 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2819): Flushing b8af53427af1873ee686b4b22616f51c 2/2 column families, dataSize=1.14 KB heapSize=2.23 KB 2023-07-22T12:38:31,522 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/meta/44d90b27cb2646e2a8f344c0895cc914 is 171, key is trslm:hdfs://localhost:33313/backupUT\x00ns3:test-16900294235332/meta:log-roll-map/1690029510460/Put/seqid=0 2023-07-22T12:38:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-07-22T12:38:31,926 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=458 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/meta/44d90b27cb2646e2a8f344c0895cc914 2023-07-22T12:38:31,932 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 44d90b27cb2646e2a8f344c0895cc914 2023-07-22T12:38:31,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/session/27212316e8204a85bde5b84e3647aff0 is 446, key is session:backup_1690029499404/session:context/1690029510886/Put/seqid=0 2023-07-22T12:38:32,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-07-22T12:38:32,339 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=712 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/session/27212316e8204a85bde5b84e3647aff0 2023-07-22T12:38:32,344 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 27212316e8204a85bde5b84e3647aff0 2023-07-22T12:38:32,345 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/meta/44d90b27cb2646e2a8f344c0895cc914 as hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/44d90b27cb2646e2a8f344c0895cc914 2023-07-22T12:38:32,350 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/44d90b27cb2646e2a8f344c0895cc914, entries=4, sequenceid=72, filesize=5.6 K 2023-07-22T12:38:32,351 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/session/27212316e8204a85bde5b84e3647aff0 as hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/27212316e8204a85bde5b84e3647aff0 2023-07-22T12:38:32,356 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/27212316e8204a85bde5b84e3647aff0, entries=2, sequenceid=72, filesize=5.4 K 2023-07-22T12:38:32,357 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1170, heapSize ~2.20 KB/2248, currentSize=0 B/0 for b8af53427af1873ee686b4b22616f51c in 839ms, sequenceid=72, compaction requested=true 2023-07-22T12:38:32,357 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2520): Flush status journal for b8af53427af1873ee686b4b22616f51c: 2023-07-22T12:38:32,357 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. for snapshot_backup_system completed. 2023-07-22T12:38:32,357 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:38:32,357 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:38:32,357 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/8b4e5904fd5541caabafed2e0a7bbecc, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/d7524c21e0884ef7a726165d97bdf81e, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/4b24f1898bb44138bbfed9c444ed2091, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/8f72b04c78fb4f5f8c4135a5820932d6, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/f1b01e23079e4b62937f7c002d957751, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/44d90b27cb2646e2a8f344c0895cc914] hfiles 2023-07-22T12:38:32,357 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (1/6): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/8b4e5904fd5541caabafed2e0a7bbecc for snapshot=snapshot_backup_system 2023-07-22T12:38:32,357 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (2/6): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/d7524c21e0884ef7a726165d97bdf81e for snapshot=snapshot_backup_system 2023-07-22T12:38:32,357 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (3/6): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/4b24f1898bb44138bbfed9c444ed2091 for snapshot=snapshot_backup_system 2023-07-22T12:38:32,357 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (4/6): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/8f72b04c78fb4f5f8c4135a5820932d6 for snapshot=snapshot_backup_system 2023-07-22T12:38:32,357 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (5/6): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/f1b01e23079e4b62937f7c002d957751 for snapshot=snapshot_backup_system 2023-07-22T12:38:32,357 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (6/6): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/44d90b27cb2646e2a8f344c0895cc914 for snapshot=snapshot_backup_system 2023-07-22T12:38:32,357 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/e7980d1347624a0e8cf8cbac22176c14, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/40fe5705ee804503bdd7677bd59e3be8, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/882f60fdcf9248459ac9b39b6e8e1f01, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/43bbdf59d84f4c98ac7bd38ac51d7e69, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/a30af91d0ecb4b06a2e7606fcf805b33, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/e19bff70c6164f2581f82c4faaec10d1, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/27212316e8204a85bde5b84e3647aff0] hfiles 2023-07-22T12:38:32,357 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (1/7): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/e7980d1347624a0e8cf8cbac22176c14 for snapshot=snapshot_backup_system 2023-07-22T12:38:32,357 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (2/7): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/40fe5705ee804503bdd7677bd59e3be8 for snapshot=snapshot_backup_system 2023-07-22T12:38:32,357 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (3/7): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/882f60fdcf9248459ac9b39b6e8e1f01 for snapshot=snapshot_backup_system 2023-07-22T12:38:32,357 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (4/7): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/43bbdf59d84f4c98ac7bd38ac51d7e69 for snapshot=snapshot_backup_system 2023-07-22T12:38:32,357 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (5/7): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/a30af91d0ecb4b06a2e7606fcf805b33 for snapshot=snapshot_backup_system 2023-07-22T12:38:32,357 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (6/7): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/e19bff70c6164f2581f82c4faaec10d1 for snapshot=snapshot_backup_system 2023-07-22T12:38:32,357 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (7/7): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/27212316e8204a85bde5b84e3647aff0 for snapshot=snapshot_backup_system 2023-07-22T12:38:32,763 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. 2023-07-22T12:38:32,763 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=51 2023-07-22T12:38:32,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.HMaster(3994): Remote procedure done, pid=51 2023-07-22T12:38:32,763 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region b8af53427af1873ee686b4b22616f51c 2023-07-22T12:38:32,765 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c 2023-07-22T12:38:32,767 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=51, resume processing ppid=50 2023-07-22T12:38:32,767 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:38:32,767 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=51, ppid=50, state=SUCCESS; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c in 1.4010 sec 2023-07-22T12:38:32,767 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:38:32,768 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:38:32,768 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:38:32,768 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:38:33,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-07-22T12:38:33,178 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:38:33,189 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:38:33,189 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:38:34,006 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:38:34,006 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 50 2023-07-22T12:38:34,007 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=50, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0530 sec 2023-07-22T12:38:35,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-07-22T12:38:35,062 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 50 completed 2023-07-22T12:38:35,063 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(115): Backup backup_1690029510908 started at 1690029515063. 2023-07-22T12:38:35,064 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(119): Backup session backup_1690029510908 has been started. 2023-07-22T12:38:35,069 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalTableBackupClient(268): For incremental backup, current table set is [test-1690029423533, ns2:test-16900294235331, ns3:test-16900294235332] 2023-07-22T12:38:35,073 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(76): StartCode 1690029480602for backupID backup_1690029510908 2023-07-22T12:38:35,073 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-07-22T12:38:35,073 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-07-22T12:38:35,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-07-22T12:38:35,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-07-22T12:38:35,074 INFO [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-07-22T12:38:35,074 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-07-22T12:38:35,075 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-07-22T12:38:35,075 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-07-22T12:38:35,076 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-07-22T12:38:35,076 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:38:35,078 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:35,078 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-07-22T12:38:35,079 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:38:35,079 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:38:35,079 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:35,079 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-07-22T12:38:35,079 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-07-22T12:38:35,079 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-07-22T12:38:35,080 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-07-22T12:38:35,080 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:38:35,080 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-07-22T12:38:35,080 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-07-22T12:38:35,080 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-07-22T12:38:35,080 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-07-22T12:38:35,080 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-07-22T12:38:35,080 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-07-22T12:38:35,081 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-07-22T12:38:35,081 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-07-22T12:38:35,081 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,39283,1690029414077' joining acquired barrier for procedure (rolllog) in zk 2023-07-22T12:38:35,084 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:35,084 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-07-22T12:38:35,084 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:35,084 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:38:35,084 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:38:35,084 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-07-22T12:38:35,084 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-07-22T12:38:35,084 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:38:35,085 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:38:35,085 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:38:35,086 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:35,086 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:38:35,086 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,39283,1690029414077' joining acquired barrier for procedure 'rolllog' on coordinator 2023-07-22T12:38:35,086 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@761d24c5[Count = 0] remaining members to acquire global barrier 2023-07-22T12:38:35,086 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-07-22T12:38:35,086 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-07-22T12:38:35,088 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:38:35,088 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-07-22T12:38:35,088 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-07-22T12:38:35,088 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-07-22T12:38:35,088 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:35,088 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-07-22T12:38:35,088 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-07-22T12:38:35,088 DEBUG [rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:35,088 INFO [rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1690029503560 highest: 1690029503560 on jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:35,088 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C39283%2C1690029414077:(num 1690029503560) roll requested 2023-07-22T12:38:35,102 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029515088, exclude list is [], retry=0 2023-07-22T12:38:35,107 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK] 2023-07-22T12:38:35,109 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029503560 with entries=10, filesize=2.37 KB; new WAL /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029515088 2023-07-22T12:38:35,109 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK]] 2023-07-22T12:38:35,109 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029503560 is not closed yet, will try archiving it next time 2023-07-22T12:38:35,111 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta:.meta(num 1690029503579) roll requested 2023-07-22T12:38:35,111 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029492145 to hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029492145 2023-07-22T12:38:35,125 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029515111.meta, exclude list is [], retry=0 2023-07-22T12:38:35,128 DEBUG [rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-07-22T12:38:35,129 INFO [rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1690029515088 on jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:35,130 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK] 2023-07-22T12:38:35,131 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029503579.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029515111.meta 2023-07-22T12:38:35,131 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:33835,DS-bd231103-555e-4ded-91ba-364f1ff937fa,DISK]] 2023-07-22T12:38:35,131 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029503579.meta is not closed yet, will try archiving it next time 2023-07-22T12:38:35,134 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029503579.meta to hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029503579.meta 2023-07-22T12:38:35,140 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-07-22T12:38:35,140 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,39283,1690029414077' in zk 2023-07-22T12:38:35,143 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-07-22T12:38:35,143 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:35,143 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:38:35,143 DEBUG [member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-07-22T12:38:35,144 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:35,144 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:38:35,144 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:38:35,145 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:38:35,145 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:38:35,145 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:38:35,145 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:35,146 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:38:35,146 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:38:35,146 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:35,147 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,39283,1690029414077': 2023-07-22T12:38:35,147 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,39283,1690029414077' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-07-22T12:38:35,147 INFO [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-07-22T12:38:35,147 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-07-22T12:38:35,147 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-07-22T12:38:35,147 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-07-22T12:38:35,147 INFO [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-07-22T12:38:35,149 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:38:35,149 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:38:35,149 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-07-22T12:38:35,149 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-07-22T12:38:35,149 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-07-22T12:38:35,149 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:38:35,149 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:38:35,149 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:38:35,149 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:38:35,149 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:38:35,149 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:38:35,149 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:35,149 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:38:35,149 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:38:35,150 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:38:35,150 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:38:35,150 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:35,151 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:35,151 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:38:35,151 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:38:35,151 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:35,155 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:35,155 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-07-22T12:38:35,156 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-07-22T12:38:35,156 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:35,156 DEBUG [(jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:38:35,156 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:38:35,156 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2023-07-22T12:38:35,156 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-07-22T12:38:35,156 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:38:35,156 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-07-22T12:38:35,156 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:38:35,156 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-07-22T12:38:35,156 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-07-22T12:38:35,156 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:38:35,156 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:38:35,157 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin(2751): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-07-22T12:38:35,157 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin(2759): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-07-22T12:38:35,157 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:38:35,157 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:38:35,257 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin(2765): Getting current status of procedure from master... 2023-07-22T12:38:35,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-07-22T12:38:35,260 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:39283=1690029480602} newestTimestamps: {jenkins-hbase3.apache.org:39283=1690029503560} 2023-07-22T12:38:35,262 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029417237.meta 2023-07-22T12:38:35,262 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029515111.meta 2023-07-22T12:38:35,262 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029503560 2023-07-22T12:38:35,262 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029515088 2023-07-22T12:38:35,262 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029438065.meta 2023-07-22T12:38:35,262 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029457350.meta 2023-07-22T12:38:35,262 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029469061.meta 2023-07-22T12:38:35,263 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029480628.meta 2023-07-22T12:38:35,263 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029492167.meta 2023-07-22T12:38:35,263 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs/jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta.1690029503579.meta 2023-07-22T12:38:35,672 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(146): Attempting to copy table info for:test-1690029423533 target: hdfs://localhost:33313/backupUT/backup_1690029510908/default/test-1690029423533 descriptor: 'test-1690029423533', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:38:35,672 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-07-22T12:38:35,674 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(151): Starting to write region info for table test-1690029423533 2023-07-22T12:38:36,079 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(158): Finished writing region info for table test-1690029423533 2023-07-22T12:38:36,488 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(146): Attempting to copy table info for:ns2:test-16900294235331 target: hdfs://localhost:33313/backupUT/backup_1690029510908/ns2/test-16900294235331 descriptor: 'ns2:test-16900294235331', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:38:36,488 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-07-22T12:38:36,490 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(151): Starting to write region info for table ns2:test-16900294235331 2023-07-22T12:38:36,895 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(158): Finished writing region info for table ns2:test-16900294235331 2023-07-22T12:38:37,305 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(146): Attempting to copy table info for:ns3:test-16900294235332 target: hdfs://localhost:33313/backupUT/backup_1690029510908/ns3/test-16900294235332 descriptor: 'ns3:test-16900294235332', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:38:37,305 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-07-22T12:38:37,307 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(151): Starting to write region info for table ns3:test-16900294235332 2023-07-22T12:38:37,718 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(158): Finished writing region info for table ns3:test-16900294235332 2023-07-22T12:38:37,723 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:33313/backupUT/.tmp/backup_1690029510908 from hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029503560;hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029492145 2023-07-22T12:38:37,724 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2808c24e to 127.0.0.1:51445 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:38:37,730 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37a43d2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:38:37,731 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:38:37,732 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36084, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:38:37,734 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-07-22T12:38:37,734 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1690029423533 2023-07-22T12:38:37,738 DEBUG [hconnection-0xf572c2b-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:38:37,739 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:50290, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:38:37,740 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1690029423533: test-1690029423533; 2023-07-22T12:38:37,740 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16900294235331 2023-07-22T12:38:37,742 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16900294235331: ns2:test-16900294235331; 2023-07-22T12:38:37,742 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns3:test-16900294235332 2023-07-22T12:38:37,743 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns3:test-16900294235332: ns3:test-16900294235332; 2023-07-22T12:38:37,743 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(694): Configuring 3 reduce partitions to match current region count for all tables 2023-07-22T12:38:37,743 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/hbase-staging/partitions_317c699a-13e6-46c0-a7fa-2ab91f624c6e 2023-07-22T12:38:38,150 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:38,150 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:38,151 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:38,151 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:39,128 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/hadoop-5377792504147655914.jar 2023-07-22T12:38:39,129 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:39,129 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:39,130 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:39,130 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:39,130 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:39,131 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:39,131 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:39,131 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:39,132 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:39,133 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:38:39,133 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:38:39,134 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:38:39,134 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:38:39,134 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:38:39,135 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:38:39,135 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:38:39,136 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:38:39,136 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:38:39,136 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:38:39,137 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:38:39,137 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:38:39,138 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:39,138 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:39,139 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:39,139 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:39,139 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:39,139 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:38:39,140 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:38:39,140 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: default:test-1690029423533,ns2:test-16900294235331,ns3:test-16900294235332 2023-07-22T12:38:39,140 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:38:39,141 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2808c24e to 127.0.0.1:51445 2023-07-22T12:38:39,141 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:38:39,142 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:38:40,141 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/hadoop-1683990905414801668.jar 2023-07-22T12:38:40,146 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-07-22T12:38:40,162 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029503560 for WAL files 2023-07-22T12:38:40,163 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029503560 2023-07-22T12:38:40,163 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029492145 for WAL files 2023-07-22T12:38:40,164 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029492145 2023-07-22T12:38:40,242 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/mapreduce.cluster.local.dir/1690029520206/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:38:40,242 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/mapreduce.cluster.local.dir/1690029520206/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:38:40,298 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029492145 for hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029492145 (-9223372036854775808:9223372036854775807) length:2574 2023-07-22T12:38:40,301 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-07-22T12:38:40,301 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-07-22T12:38:40,352 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029503560 for hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/WALs/jenkins-hbase3.apache.org,39283,1690029414077/jenkins-hbase3.apache.org%2C39283%2C1690029414077.1690029503560 (-9223372036854775808:9223372036854775807) length:2432 2023-07-22T12:38:40,356 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-07-22T12:38:40,356 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-07-22T12:38:41,281 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:33313/backupUT 2023-07-22T12:38:41,281 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1690029510908 2023-07-22T12:38:41,281 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-07-22T12:38:41,286 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:33313/backupUT/.tmp/backup_1690029510908, hdfs://localhost:33313/backupUT] 2023-07-22T12:38:41,308 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-07-22T12:38:41,887 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-07-22T12:38:41,889 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1690029510908 2023-07-22T12:38:41,889 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 0 bytes copied." 2023-07-22T12:38:41,889 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local628297045_0013 completed: true true 2023-07-22T12:38:41,892 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=6698165 FILE: Number of bytes written=15943483 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=335294 HDFS: Number of bytes written=2484404 HDFS: Number of read operations=1146 HDFS: Number of large read operations=0 HDFS: Number of write operations=429 Map-Reduce Framework Map input records=2 Map output records=0 Input split bytes=342 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1394606080 File Input Format Counters Bytes Read=486 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=0 Bytes Expected=0 Files Copied=1 DIR_COPY=1 2023-07-22T12:38:41,892 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:33313/backupUT/.tmp/backup_1690029510908 to hdfs://localhost:33313/backupUT finished. 2023-07-22T12:38:41,902 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(433): deleted 0 original bulkload rows 2023-07-22T12:38:41,902 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029510908 2023-07-22T12:38:41,906 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029487980 to construct the dependency. 2023-07-22T12:38:41,906 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029487980 2023-07-22T12:38:41,909 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029487980/.backup.manifest 2023-07-22T12:38:41,909 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029487980BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:41,909 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029476431 to construct the dependency. 2023-07-22T12:38:41,909 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029476431 2023-07-22T12:38:41,912 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029476431/.backup.manifest 2023-07-22T12:38:41,912 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029476431BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:41,912 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029464859 to construct the dependency. 2023-07-22T12:38:41,912 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029464859 2023-07-22T12:38:41,914 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029464859/.backup.manifest 2023-07-22T12:38:41,914 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029464859BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:41,914 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029453150 to construct the dependency. 2023-07-22T12:38:41,914 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029453150 2023-07-22T12:38:41,916 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029453150/.backup.manifest 2023-07-22T12:38:41,916 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029453150BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:41,916 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(340): Got 6 ancestors for the current backup. 2023-07-22T12:38:41,916 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029510908 2023-07-22T12:38:41,919 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029487980 to construct the dependency. 2023-07-22T12:38:41,919 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029487980 2023-07-22T12:38:41,921 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029487980/.backup.manifest 2023-07-22T12:38:41,922 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029487980BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:41,922 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029476431 to construct the dependency. 2023-07-22T12:38:41,922 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029476431 2023-07-22T12:38:41,924 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029476431/.backup.manifest 2023-07-22T12:38:41,924 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029476431BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:41,924 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029464859 to construct the dependency. 2023-07-22T12:38:41,924 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029464859 2023-07-22T12:38:41,926 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029464859/.backup.manifest 2023-07-22T12:38:41,926 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029464859BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:41,926 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029453150 to construct the dependency. 2023-07-22T12:38:41,926 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029453150 2023-07-22T12:38:41,928 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029453150/.backup.manifest 2023-07-22T12:38:41,928 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029453150BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:41,928 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(340): Got 6 ancestors for the current backup. 2023-07-22T12:38:42,335 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33313/backupUT/backup_1690029510908/.backup.manifest 2023-07-22T12:38:42,335 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029510908 2023-07-22T12:38:42,337 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029487980 to construct the dependency. 2023-07-22T12:38:42,338 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029487980 2023-07-22T12:38:42,340 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029487980/.backup.manifest 2023-07-22T12:38:42,340 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029487980BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:42,340 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029476431 to construct the dependency. 2023-07-22T12:38:42,340 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029476431 2023-07-22T12:38:42,342 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029476431/.backup.manifest 2023-07-22T12:38:42,342 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029476431BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:42,342 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029464859 to construct the dependency. 2023-07-22T12:38:42,342 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029464859 2023-07-22T12:38:42,344 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029464859/.backup.manifest 2023-07-22T12:38:42,344 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029464859BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:42,344 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029453150 to construct the dependency. 2023-07-22T12:38:42,344 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029453150 2023-07-22T12:38:42,346 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029453150/.backup.manifest 2023-07-22T12:38:42,346 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029453150BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:42,346 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(340): Got 6 ancestors for the current backup. 2023-07-22T12:38:42,346 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029510908 2023-07-22T12:38:42,348 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029487980 to construct the dependency. 2023-07-22T12:38:42,348 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029487980 2023-07-22T12:38:42,350 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029487980/.backup.manifest 2023-07-22T12:38:42,350 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029487980BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:42,350 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029476431 to construct the dependency. 2023-07-22T12:38:42,350 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029476431 2023-07-22T12:38:42,352 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029476431/.backup.manifest 2023-07-22T12:38:42,352 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029476431BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:42,352 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029464859 to construct the dependency. 2023-07-22T12:38:42,352 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029464859 2023-07-22T12:38:42,354 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029464859/.backup.manifest 2023-07-22T12:38:42,354 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029464859BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:42,354 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029453150 to construct the dependency. 2023-07-22T12:38:42,354 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029453150 2023-07-22T12:38:42,356 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029453150/.backup.manifest 2023-07-22T12:38:42,356 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029453150BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:42,356 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(340): Got 6 ancestors for the current backup. 2023-07-22T12:38:42,762 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33313/backupUT/backup_1690029510908/.backup.manifest 2023-07-22T12:38:42,762 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029510908 2023-07-22T12:38:42,765 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029487980 to construct the dependency. 2023-07-22T12:38:42,765 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029487980 2023-07-22T12:38:42,768 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029487980/.backup.manifest 2023-07-22T12:38:42,768 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029487980BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:42,768 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029476431 to construct the dependency. 2023-07-22T12:38:42,768 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029476431 2023-07-22T12:38:42,770 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029476431/.backup.manifest 2023-07-22T12:38:42,770 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029476431BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:42,770 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029464859 to construct the dependency. 2023-07-22T12:38:42,770 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029464859 2023-07-22T12:38:42,773 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029464859/.backup.manifest 2023-07-22T12:38:42,773 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029464859BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:42,773 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029453150 to construct the dependency. 2023-07-22T12:38:42,773 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029453150 2023-07-22T12:38:42,775 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029453150/.backup.manifest 2023-07-22T12:38:42,775 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029453150BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:42,775 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(340): Got 6 ancestors for the current backup. 2023-07-22T12:38:42,775 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029510908 2023-07-22T12:38:42,777 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029487980 to construct the dependency. 2023-07-22T12:38:42,778 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029487980 2023-07-22T12:38:42,780 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029487980/.backup.manifest 2023-07-22T12:38:42,780 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029487980BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:42,780 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029476431 to construct the dependency. 2023-07-22T12:38:42,780 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029476431 2023-07-22T12:38:42,781 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029476431/.backup.manifest 2023-07-22T12:38:42,781 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029476431BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:42,781 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029464859 to construct the dependency. 2023-07-22T12:38:42,781 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029464859 2023-07-22T12:38:42,783 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029464859/.backup.manifest 2023-07-22T12:38:42,783 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029464859BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:42,783 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029453150 to construct the dependency. 2023-07-22T12:38:42,783 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029453150 2023-07-22T12:38:42,785 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029453150/.backup.manifest 2023-07-22T12:38:42,785 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029453150BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:42,785 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(340): Got 6 ancestors for the current backup. 2023-07-22T12:38:43,190 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33313/backupUT/backup_1690029510908/.backup.manifest 2023-07-22T12:38:43,190 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029510908 2023-07-22T12:38:43,193 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029487980 to construct the dependency. 2023-07-22T12:38:43,193 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029487980 2023-07-22T12:38:43,195 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029487980/.backup.manifest 2023-07-22T12:38:43,195 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029487980BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:43,195 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029476431 to construct the dependency. 2023-07-22T12:38:43,196 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029476431 2023-07-22T12:38:43,198 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029476431/.backup.manifest 2023-07-22T12:38:43,198 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029476431BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:43,198 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029464859 to construct the dependency. 2023-07-22T12:38:43,198 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029464859 2023-07-22T12:38:43,200 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029464859/.backup.manifest 2023-07-22T12:38:43,200 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029464859BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:43,200 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33313/backupUT/backup_1690029453150 to construct the dependency. 2023-07-22T12:38:43,200 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33313/backupUT/backup_1690029453150 2023-07-22T12:38:43,202 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1690029453150/.backup.manifest 2023-07-22T12:38:43,202 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1690029453150BackupDir=hdfs://localhost:33313/backupUT} 2023-07-22T12:38:43,202 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManager(340): Got 6 ancestors for the current backup. 2023-07-22T12:38:43,607 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33313/backupUT/backup_1690029510908/.backup.manifest 2023-07-22T12:38:43,607 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(358): Backup backup_1690029510908 finished: type=INCREMENTAL,tablelist=test-1690029423533;ns2:test-16900294235331;ns3:test-16900294235332,targetRootDir=hdfs://localhost:33313/backupUT,startts=1690029515063,completets=1690029521902,bytescopied=0 2023-07-22T12:38:43,607 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:38:43,612 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:38:43,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:38:43,615 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:38:43,617 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:38:43,620 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.TableBackupClient(376): Backup backup_1690029510908 completed. 2023-07-22T12:38:43,630 ERROR [Listener at jenkins-hbase3.apache.org/34965 {}] backup.TestBackupMultipleDeletes(130): Delete backupIdInc2 2023-07-22T12:38:43,638 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:38:43,646 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:38:43,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029523647 (current time:1690029523647). 2023-07-22T12:38:43,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:38:43,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:38:43,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x15aafe04 to 127.0.0.1:51445 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:38:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@204d8e86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:38:43,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:38:43,659 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:47220, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:38:43,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x15aafe04 to 127.0.0.1:51445 2023-07-22T12:38:43,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:38:43,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:38:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure2.ProcedureExecutor(1033): Stored pid=52, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:38:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-07-22T12:38:43,663 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:38:43,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-07-22T12:38:43,664 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:38:43,666 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:38:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-07-22T12:38:43,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-07-22T12:38:44,072 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:38:44,072 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c}] 2023-07-22T12:38:44,074 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c 2023-07-22T12:38:44,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39283 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=53 2023-07-22T12:38:44,226 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. 2023-07-22T12:38:44,226 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2819): Flushing b8af53427af1873ee686b4b22616f51c 2/2 column families, dataSize=1.75 KB heapSize=3.05 KB 2023-07-22T12:38:44,230 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/meta/9e6a4ccee5334e62bda59100f05e6b43 is 171, key is trslm:hdfs://localhost:33313/backupUT\x00ns2:test-16900294235331/meta:log-roll-map/1690029521896/Put/seqid=0 2023-07-22T12:38:44,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-07-22T12:38:44,635 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=758 B at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/meta/9e6a4ccee5334e62bda59100f05e6b43 2023-07-22T12:38:44,640 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 9e6a4ccee5334e62bda59100f05e6b43 2023-07-22T12:38:44,643 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/session/d2eaeb75a51f492893c7020dc1cbb5c5 is 461, key is session:backup_1690029510908/session:context/1690029523616/Put/seqid=0 2023-07-22T12:38:44,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-07-22T12:38:45,047 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.01 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/session/d2eaeb75a51f492893c7020dc1cbb5c5 2023-07-22T12:38:45,052 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: d2eaeb75a51f492893c7020dc1cbb5c5 2023-07-22T12:38:45,053 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/meta/9e6a4ccee5334e62bda59100f05e6b43 as hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/9e6a4ccee5334e62bda59100f05e6b43 2023-07-22T12:38:45,057 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/9e6a4ccee5334e62bda59100f05e6b43, entries=6, sequenceid=84, filesize=5.8 K 2023-07-22T12:38:45,058 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/session/d2eaeb75a51f492893c7020dc1cbb5c5 as hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/d2eaeb75a51f492893c7020dc1cbb5c5 2023-07-22T12:38:45,063 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/d2eaeb75a51f492893c7020dc1cbb5c5, entries=2, sequenceid=84, filesize=5.4 K 2023-07-22T12:38:45,064 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(3022): Finished flush of dataSize ~1.75 KB/1792, heapSize ~3.02 KB/3096, currentSize=0 B/0 for b8af53427af1873ee686b4b22616f51c in 837ms, sequenceid=84, compaction requested=true 2023-07-22T12:38:45,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2520): Flush status journal for b8af53427af1873ee686b4b22616f51c: 2023-07-22T12:38:45,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. for snapshot_backup_system completed. 2023-07-22T12:38:45,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:38:45,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:38:45,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/8b4e5904fd5541caabafed2e0a7bbecc, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/d7524c21e0884ef7a726165d97bdf81e, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/4b24f1898bb44138bbfed9c444ed2091, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/8f72b04c78fb4f5f8c4135a5820932d6, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/f1b01e23079e4b62937f7c002d957751, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/44d90b27cb2646e2a8f344c0895cc914, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/9e6a4ccee5334e62bda59100f05e6b43] hfiles 2023-07-22T12:38:45,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (1/7): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/8b4e5904fd5541caabafed2e0a7bbecc for snapshot=snapshot_backup_system 2023-07-22T12:38:45,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (2/7): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/d7524c21e0884ef7a726165d97bdf81e for snapshot=snapshot_backup_system 2023-07-22T12:38:45,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (3/7): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/4b24f1898bb44138bbfed9c444ed2091 for snapshot=snapshot_backup_system 2023-07-22T12:38:45,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (4/7): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/8f72b04c78fb4f5f8c4135a5820932d6 for snapshot=snapshot_backup_system 2023-07-22T12:38:45,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (5/7): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/f1b01e23079e4b62937f7c002d957751 for snapshot=snapshot_backup_system 2023-07-22T12:38:45,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (6/7): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/44d90b27cb2646e2a8f344c0895cc914 for snapshot=snapshot_backup_system 2023-07-22T12:38:45,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (7/7): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/9e6a4ccee5334e62bda59100f05e6b43 for snapshot=snapshot_backup_system 2023-07-22T12:38:45,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/e7980d1347624a0e8cf8cbac22176c14, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/40fe5705ee804503bdd7677bd59e3be8, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/882f60fdcf9248459ac9b39b6e8e1f01, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/43bbdf59d84f4c98ac7bd38ac51d7e69, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/a30af91d0ecb4b06a2e7606fcf805b33, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/e19bff70c6164f2581f82c4faaec10d1, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/27212316e8204a85bde5b84e3647aff0, hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/d2eaeb75a51f492893c7020dc1cbb5c5] hfiles 2023-07-22T12:38:45,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (1/8): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/e7980d1347624a0e8cf8cbac22176c14 for snapshot=snapshot_backup_system 2023-07-22T12:38:45,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (2/8): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/40fe5705ee804503bdd7677bd59e3be8 for snapshot=snapshot_backup_system 2023-07-22T12:38:45,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (3/8): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/882f60fdcf9248459ac9b39b6e8e1f01 for snapshot=snapshot_backup_system 2023-07-22T12:38:45,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (4/8): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/43bbdf59d84f4c98ac7bd38ac51d7e69 for snapshot=snapshot_backup_system 2023-07-22T12:38:45,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (5/8): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/a30af91d0ecb4b06a2e7606fcf805b33 for snapshot=snapshot_backup_system 2023-07-22T12:38:45,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (6/8): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/e19bff70c6164f2581f82c4faaec10d1 for snapshot=snapshot_backup_system 2023-07-22T12:38:45,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (7/8): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/27212316e8204a85bde5b84e3647aff0 for snapshot=snapshot_backup_system 2023-07-22T12:38:45,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (8/8): hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/d2eaeb75a51f492893c7020dc1cbb5c5 for snapshot=snapshot_backup_system 2023-07-22T12:38:45,470 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. 2023-07-22T12:38:45,470 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=53 2023-07-22T12:38:45,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.HMaster(3994): Remote procedure done, pid=53 2023-07-22T12:38:45,470 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region b8af53427af1873ee686b4b22616f51c 2023-07-22T12:38:45,472 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c 2023-07-22T12:38:45,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=53, resume processing ppid=52 2023-07-22T12:38:45,474 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:38:45,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=53, ppid=52, state=SUCCESS; SnapshotRegionProcedure b8af53427af1873ee686b4b22616f51c in 1.4000 sec 2023-07-22T12:38:45,474 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:38:45,475 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:38:45,475 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:38:45,475 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:38:45,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-07-22T12:38:45,885 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:38:45,896 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:38:45,897 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:38:46,717 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:38:46,717 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-07-22T12:38:46,718 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=52, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0560 sec 2023-07-22T12:38:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-07-22T12:38:47,771 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 52 completed 2023-07-22T12:38:47,777 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupAdminImpl(227): Deleting backup backup_1690029464859 ... 2023-07-22T12:38:47,778 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(435): Delete log files: .backup.manifest 2023-07-22T12:38:47,779 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(435): Delete log files: _SUCCESS 2023-07-22T12:38:47,779 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(435): Delete log files: default 2023-07-22T12:38:47,780 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(435): Delete log files: ns2 2023-07-22T12:38:47,781 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(443): Trying to cleanup up target dir : backup_1690029464859 2023-07-22T12:38:47,781 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(458): No data has been found in hdfs://localhost:33313/backupUT/backup_1690029464859/default/test-1690029423533. 2023-07-22T12:38:47,782 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(465): hdfs://localhost:33313/backupUT/backup_1690029464859/default is empty, remove it. 2023-07-22T12:38:47,783 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(458): No data has been found in hdfs://localhost:33313/backupUT/backup_1690029464859/ns2/test-16900294235331. 2023-07-22T12:38:47,784 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(465): hdfs://localhost:33313/backupUT/backup_1690029464859/ns2 is empty, remove it. 2023-07-22T12:38:47,800 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupAdminImpl(309): GetAffectedBackupInfos for: backup_1690029464859 table=test-1690029423533 2023-07-22T12:38:47,804 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupAdminImpl(326): GetAffectedBackupInfos for: backup_1690029464859 table=test-1690029423533 added backup_1690029510908 tables={test-1690029423533,ns2:test-16900294235331,ns3:test-16900294235332} 2023-07-22T12:38:47,804 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupAdminImpl(326): GetAffectedBackupInfos for: backup_1690029464859 table=test-1690029423533 added backup_1690029487980 tables={test-1690029423533,ns2:test-16900294235331} 2023-07-22T12:38:47,804 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupAdminImpl(326): GetAffectedBackupInfos for: backup_1690029464859 table=test-1690029423533 added backup_1690029476431 tables={test-1690029423533,ns2:test-16900294235331} 2023-07-22T12:38:47,804 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupAdminImpl(287): Remove test-1690029423533 from backup_1690029510908 tables={test-1690029423533,ns2:test-16900294235331,ns3:test-16900294235332} 2023-07-22T12:38:47,807 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupAdminImpl(354): Cleaning up backup data at hdfs://localhost:33313/backupUT/backup_1690029510908/default/test-1690029423533 done. 2023-07-22T12:38:47,807 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupAdminImpl(287): Remove test-1690029423533 from backup_1690029487980 tables={test-1690029423533,ns2:test-16900294235331} 2023-07-22T12:38:47,809 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupAdminImpl(354): Cleaning up backup data at hdfs://localhost:33313/backupUT/backup_1690029487980/default/test-1690029423533 done. 2023-07-22T12:38:47,809 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupAdminImpl(287): Remove test-1690029423533 from backup_1690029476431 tables={test-1690029423533,ns2:test-16900294235331} 2023-07-22T12:38:47,811 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupAdminImpl(354): Cleaning up backup data at hdfs://localhost:33313/backupUT/backup_1690029476431/default/test-1690029423533 done. 2023-07-22T12:38:47,814 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupAdminImpl(309): GetAffectedBackupInfos for: backup_1690029464859 table=ns2:test-16900294235331 2023-07-22T12:38:47,819 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupAdminImpl(326): GetAffectedBackupInfos for: backup_1690029464859 table=ns2:test-16900294235331 added backup_1690029510908 tables={ns2:test-16900294235331,ns3:test-16900294235332} 2023-07-22T12:38:47,819 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupAdminImpl(326): GetAffectedBackupInfos for: backup_1690029464859 table=ns2:test-16900294235331 added backup_1690029487980 tables={ns2:test-16900294235331} 2023-07-22T12:38:47,819 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupAdminImpl(326): GetAffectedBackupInfos for: backup_1690029464859 table=ns2:test-16900294235331 added backup_1690029476431 tables={ns2:test-16900294235331} 2023-07-22T12:38:47,819 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupAdminImpl(287): Remove ns2:test-16900294235331 from backup_1690029510908 tables={ns2:test-16900294235331,ns3:test-16900294235332} 2023-07-22T12:38:47,822 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupAdminImpl(354): Cleaning up backup data at hdfs://localhost:33313/backupUT/backup_1690029510908/ns2/test-16900294235331 done. 2023-07-22T12:38:47,822 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupAdminImpl(287): Remove ns2:test-16900294235331 from backup_1690029487980 tables={ns2:test-16900294235331} 2023-07-22T12:38:47,822 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupAdminImpl(293): Delete backup info backup_1690029487980 2023-07-22T12:38:47,825 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(435): Delete log files: .backup.manifest 2023-07-22T12:38:47,826 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(435): Delete log files: _SUCCESS 2023-07-22T12:38:47,827 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(435): Delete log files: default 2023-07-22T12:38:47,827 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(435): Delete log files: ns2 2023-07-22T12:38:47,828 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(443): Trying to cleanup up target dir : backup_1690029487980 2023-07-22T12:38:47,828 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(458): No data has been found in hdfs://localhost:33313/backupUT/backup_1690029487980/ns2/test-16900294235331. 2023-07-22T12:38:47,829 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(465): hdfs://localhost:33313/backupUT/backup_1690029487980/ns2 is empty, remove it. 2023-07-22T12:38:47,830 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupAdminImpl(287): Remove ns2:test-16900294235331 from backup_1690029476431 tables={ns2:test-16900294235331} 2023-07-22T12:38:47,830 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupAdminImpl(293): Delete backup info backup_1690029476431 2023-07-22T12:38:47,832 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(435): Delete log files: .backup.manifest 2023-07-22T12:38:47,832 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(435): Delete log files: _SUCCESS 2023-07-22T12:38:47,833 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(435): Delete log files: default 2023-07-22T12:38:47,834 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(435): Delete log files: ns2 2023-07-22T12:38:47,834 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(443): Trying to cleanup up target dir : backup_1690029476431 2023-07-22T12:38:47,835 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(458): No data has been found in hdfs://localhost:33313/backupUT/backup_1690029476431/ns2/test-16900294235331. 2023-07-22T12:38:47,835 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.BackupUtils(465): hdfs://localhost:33313/backupUT/backup_1690029476431/ns2 is empty, remove it. 2023-07-22T12:38:47,837 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupAdminImpl(269): 0 bulk loaded files out of 0 were deleted 2023-07-22T12:38:47,838 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(433): deleted 0 original bulkload rows 2023-07-22T12:38:47,839 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupAdminImpl(276): Delete backup backup_1690029464859 completed. 2023-07-22T12:38:47,844 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(1014): test-1690029423533 2023-07-22T12:38:47,844 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(1014): ns2:test-16900294235331 2023-07-22T12:38:47,844 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(1014): ns3:test-16900294235332 2023-07-22T12:38:47,846 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:38:47,851 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:38:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33617 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:38:47,854 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:38:47,854 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:38:47,856 ERROR [Listener at jenkins-hbase3.apache.org/34965 {}] backup.TestBackupMultipleDeletes(132): Delete backupIdInc2 done 2023-07-22T12:38:47,870 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:38:47,870 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x548d09bd to 127.0.0.1:51445 2023-07-22T12:38:47,871 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:38:47,894 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] hbase.ResourceChecker(175): after: backup.TestBackupMultipleDeletes#testBackupMultipleDeletes Thread=590 (was 584) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1692848380_16 at /127.0.0.1:34508 [Receiving block BP-1667261202-172.31.12.81-1690029407680:blk_1073741980_1156] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/cluster_a7b0bb94-85a8-a89c-f1a4-3863f52e4e30/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1667261202-172.31.12.81-1690029407680:blk_1073741980_1156, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1667261202-172.31.12.81-1690029407680:blk_1073741979_1155, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0xfc46a9a-shared-pool-31 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/cluster_a7b0bb94-85a8-a89c-f1a4-3863f52e4e30/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: (jenkins-hbase3.apache.org,33617,1690029410929)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: member: 'jenkins-hbase3.apache.org,39283,1690029414077' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1173140135_16 at /127.0.0.1:41848 [Waiting for operation #3] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1766121934_16 at /127.0.0.1:41866 [Waiting for operation #11] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1692848380_16 at /127.0.0.1:34506 [Receiving block BP-1667261202-172.31.12.81-1690029407680:blk_1073741979_1155] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,39283,1690029414077)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1173140135_16 at /127.0.0.1:41766 [Waiting for operation #45] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: region-location-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=677 (was 656) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=115 (was 271), ProcessCount=169 (was 169), AvailableMemoryMB=15951 (was 17181) 2023-07-22T12:38:47,894 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] hbase.ResourceChecker(130): Thread=590 is superior to 500 2023-07-22T12:38:47,903 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-07-22T12:38:47,903 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:38:47,903 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x43801001 to 127.0.0.1:51445 2023-07-22T12:38:47,903 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:38:47,903 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-07-22T12:38:47,903 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] util.JVMClusterUtil(257): Found active master hash=1319493288, stopped=false 2023-07-22T12:38:47,904 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] master.ServerManager(910): Cluster shutdown requested of master=jenkins-hbase3.apache.org,33617,1690029410929 2023-07-22T12:38:47,906 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2023-07-22T12:38:47,907 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:38:47,907 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2023-07-22T12:38:47,907 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:38:47,907 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] procedure2.ProcedureExecutor(631): Stopping 2023-07-22T12:38:47,907 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-07-22T12:38:47,907 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-07-22T12:38:47,907 DEBUG [Listener at jenkins-hbase3.apache.org/34965 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:38:47,908 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] regionserver.HRegionServer(2513): ***** STOPPING region server 'jenkins-hbase3.apache.org,39283,1690029414077' ***** 2023-07-22T12:38:47,908 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] regionserver.HRegionServer(2527): STOPPED: Shutdown requested 2023-07-22T12:38:47,908 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-07-22T12:38:47,925 INFO [RS:0;jenkins-hbase3:39283 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@2234bbbc{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-07-22T12:38:47,931 INFO [RS:0;jenkins-hbase3:39283 {}] server.AbstractConnector(383): Stopped ServerConnector@67109812{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-07-22T12:38:47,931 INFO [RS:0;jenkins-hbase3:39283 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-07-22T12:38:47,932 INFO [RS:0;jenkins-hbase3:39283 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@f16de87{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-07-22T12:38:47,933 INFO [RS:0;jenkins-hbase3:39283 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@44c954d0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/hadoop.log.dir/,STOPPED} 2023-07-22T12:38:47,936 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.HeapMemoryManager(220): Stopping 2023-07-22T12:38:47,936 INFO [RS:0;jenkins-hbase3:39283 {}] flush.RegionServerFlushTableProcedureManager(117): Stopping region server flush procedure manager gracefully. 2023-07-22T12:38:47,936 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-07-22T12:38:47,936 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-07-22T12:38:47,936 INFO [RS:0;jenkins-hbase3:39283 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-07-22T12:38:47,937 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(3524): Received CLOSE for 626922efc5eebf51b8bc065f9cae11de 2023-07-22T12:38:47,937 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(3524): Received CLOSE for 080805bc29553da1f6156bd53ce64200 2023-07-22T12:38:47,937 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(3524): Received CLOSE for c29ae4e4c3e8a42ff3c19bb2bd28710f 2023-07-22T12:38:47,937 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(3524): Received CLOSE for 4d15624c81ab8d3881a6ef2c42fdbab5 2023-07-22T12:38:47,937 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(3524): Received CLOSE for e1879fe3670023eba374a8cb2f79a944 2023-07-22T12:38:47,937 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(3524): Received CLOSE for b8af53427af1873ee686b4b22616f51c 2023-07-22T12:38:47,937 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(3524): Received CLOSE for dce7657497bf6b1da7900abd4083b1b8 2023-07-22T12:38:47,937 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:47,938 INFO [RS:0;jenkins-hbase3:39283 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:38:47,938 DEBUG [RS:0;jenkins-hbase3:39283 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:38:47,938 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 626922efc5eebf51b8bc065f9cae11de, disabling compactions & flushes 2023-07-22T12:38:47,938 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-07-22T12:38:47,938 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-16900294235331,,1690029426287.626922efc5eebf51b8bc065f9cae11de. 2023-07-22T12:38:47,938 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-07-22T12:38:47,938 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16900294235331,,1690029426287.626922efc5eebf51b8bc065f9cae11de. 2023-07-22T12:38:47,938 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-07-22T12:38:47,938 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16900294235331,,1690029426287.626922efc5eebf51b8bc065f9cae11de. after waiting 0 ms 2023-07-22T12:38:47,938 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(3524): Received CLOSE for 1588230740 2023-07-22T12:38:47,938 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16900294235331,,1690029426287.626922efc5eebf51b8bc065f9cae11de. 2023-07-22T12:38:47,938 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-07-22T12:38:47,938 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 626922efc5eebf51b8bc065f9cae11de 1/1 column families, dataSize=7.13 KB heapSize=21.91 KB 2023-07-22T12:38:47,938 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(1579): Waiting on 8 regions to close 2023-07-22T12:38:47,938 DEBUG [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(1583): Online Regions={626922efc5eebf51b8bc065f9cae11de=ns2:test-16900294235331,,1690029426287.626922efc5eebf51b8bc065f9cae11de., 080805bc29553da1f6156bd53ce64200=ns4:test-16900294235333,,1690029429610.080805bc29553da1f6156bd53ce64200., c29ae4e4c3e8a42ff3c19bb2bd28710f=ns3:test-16900294235332,,1690029427482.c29ae4e4c3e8a42ff3c19bb2bd28710f., 4d15624c81ab8d3881a6ef2c42fdbab5=backup:system_bulk,,1690029431661.4d15624c81ab8d3881a6ef2c42fdbab5., e1879fe3670023eba374a8cb2f79a944=hbase:namespace,,1690029417642.e1879fe3670023eba374a8cb2f79a944., b8af53427af1873ee686b4b22616f51c=backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c., dce7657497bf6b1da7900abd4083b1b8=test-1690029423533,,1690029424029.dce7657497bf6b1da7900abd4083b1b8., 1588230740=hbase:meta,,1.1588230740} 2023-07-22T12:38:47,943 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-07-22T12:38:47,943 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-07-22T12:38:47,943 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-07-22T12:38:47,943 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-07-22T12:38:47,943 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-07-22T12:38:47,943 DEBUG [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(1609): Waiting on 080805bc29553da1f6156bd53ce64200, 1588230740, 4d15624c81ab8d3881a6ef2c42fdbab5, 626922efc5eebf51b8bc065f9cae11de, b8af53427af1873ee686b4b22616f51c, c29ae4e4c3e8a42ff3c19bb2bd28710f, dce7657497bf6b1da7900abd4083b1b8, e1879fe3670023eba374a8cb2f79a944 2023-07-22T12:38:47,943 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=9.25 KB heapSize=16.20 KB 2023-07-22T12:38:47,952 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/ns2/test-16900294235331/626922efc5eebf51b8bc065f9cae11de/.tmp/f/62917511dbaa41a7a748ab1e1cbd229b is 42, key is row-t1100/f:q1/1690029487902/Put/seqid=0 2023-07-22T12:38:47,961 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.13 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/ns2/test-16900294235331/626922efc5eebf51b8bc065f9cae11de/.tmp/f/62917511dbaa41a7a748ab1e1cbd229b 2023-07-22T12:38:47,967 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 62917511dbaa41a7a748ab1e1cbd229b 2023-07-22T12:38:47,968 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/ns2/test-16900294235331/626922efc5eebf51b8bc065f9cae11de/.tmp/f/62917511dbaa41a7a748ab1e1cbd229b as hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/ns2/test-16900294235331/626922efc5eebf51b8bc065f9cae11de/f/62917511dbaa41a7a748ab1e1cbd229b 2023-07-22T12:38:47,977 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/ns2/test-16900294235331/626922efc5eebf51b8bc065f9cae11de/f/62917511dbaa41a7a748ab1e1cbd229b, entries=198, sequenceid=304, filesize=12.6 K 2023-07-22T12:38:47,978 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~7.13 KB/7304, heapSize ~21.89 KB/22416, currentSize=0 B/0 for 626922efc5eebf51b8bc065f9cae11de in 40ms, sequenceid=304, compaction requested=false 2023-07-22T12:38:47,979 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/hbase/meta/1588230740/.tmp/info/a8225522ff66486aa8f3ccd91b399aab is 159, key is ns2:test-16900294235331,,1690029426287.626922efc5eebf51b8bc065f9cae11de./info:regioninfo/1690029427076/Put/seqid=0 2023-07-22T12:38:47,980 DEBUG [StoreFileCloser-626922efc5eebf51b8bc065f9cae11de-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 46975ccb8d55419c99d458bac3865a94 2023-07-22T12:38:47,985 DEBUG [StoreFileCloser-626922efc5eebf51b8bc065f9cae11de-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 62917511dbaa41a7a748ab1e1cbd229b 2023-07-22T12:38:47,993 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/ns2/test-16900294235331/626922efc5eebf51b8bc065f9cae11de/recovered.edits/307.seqid, newMaxSeqId=307, maxSeqId=1 2023-07-22T12:38:47,993 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.55 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/hbase/meta/1588230740/.tmp/info/a8225522ff66486aa8f3ccd91b399aab 2023-07-22T12:38:47,994 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:38:47,995 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-16900294235331,,1690029426287.626922efc5eebf51b8bc065f9cae11de. 2023-07-22T12:38:47,995 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 626922efc5eebf51b8bc065f9cae11de: 2023-07-22T12:38:47,995 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-16900294235331,,1690029426287.626922efc5eebf51b8bc065f9cae11de. 2023-07-22T12:38:47,995 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 080805bc29553da1f6156bd53ce64200, disabling compactions & flushes 2023-07-22T12:38:47,995 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-16900294235333,,1690029429610.080805bc29553da1f6156bd53ce64200. 2023-07-22T12:38:47,995 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16900294235333,,1690029429610.080805bc29553da1f6156bd53ce64200. 2023-07-22T12:38:47,996 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16900294235333,,1690029429610.080805bc29553da1f6156bd53ce64200. after waiting 0 ms 2023-07-22T12:38:47,996 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16900294235333,,1690029429610.080805bc29553da1f6156bd53ce64200. 2023-07-22T12:38:47,999 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/ns4/test-16900294235333/080805bc29553da1f6156bd53ce64200/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-07-22T12:38:48,000 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:38:48,000 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-16900294235333,,1690029429610.080805bc29553da1f6156bd53ce64200. 2023-07-22T12:38:48,000 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 080805bc29553da1f6156bd53ce64200: 2023-07-22T12:38:48,000 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a8225522ff66486aa8f3ccd91b399aab 2023-07-22T12:38:48,000 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-16900294235333,,1690029429610.080805bc29553da1f6156bd53ce64200. 2023-07-22T12:38:48,000 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing c29ae4e4c3e8a42ff3c19bb2bd28710f, disabling compactions & flushes 2023-07-22T12:38:48,000 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-16900294235332,,1690029427482.c29ae4e4c3e8a42ff3c19bb2bd28710f. 2023-07-22T12:38:48,000 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16900294235332,,1690029427482.c29ae4e4c3e8a42ff3c19bb2bd28710f. 2023-07-22T12:38:48,000 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16900294235332,,1690029427482.c29ae4e4c3e8a42ff3c19bb2bd28710f. after waiting 0 ms 2023-07-22T12:38:48,000 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16900294235332,,1690029427482.c29ae4e4c3e8a42ff3c19bb2bd28710f. 2023-07-22T12:38:48,006 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/ns3/test-16900294235332/c29ae4e4c3e8a42ff3c19bb2bd28710f/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2023-07-22T12:38:48,007 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:38:48,008 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-16900294235332,,1690029427482.c29ae4e4c3e8a42ff3c19bb2bd28710f. 2023-07-22T12:38:48,008 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for c29ae4e4c3e8a42ff3c19bb2bd28710f: 2023-07-22T12:38:48,008 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-16900294235332,,1690029427482.c29ae4e4c3e8a42ff3c19bb2bd28710f. 2023-07-22T12:38:48,009 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 4d15624c81ab8d3881a6ef2c42fdbab5, disabling compactions & flushes 2023-07-22T12:38:48,009 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1690029431661.4d15624c81ab8d3881a6ef2c42fdbab5. 2023-07-22T12:38:48,009 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1690029431661.4d15624c81ab8d3881a6ef2c42fdbab5. 2023-07-22T12:38:48,009 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1690029431661.4d15624c81ab8d3881a6ef2c42fdbab5. after waiting 0 ms 2023-07-22T12:38:48,009 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1690029431661.4d15624c81ab8d3881a6ef2c42fdbab5. 2023-07-22T12:38:48,017 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system_bulk/4d15624c81ab8d3881a6ef2c42fdbab5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-07-22T12:38:48,018 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:38:48,018 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1690029431661.4d15624c81ab8d3881a6ef2c42fdbab5. 2023-07-22T12:38:48,018 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 4d15624c81ab8d3881a6ef2c42fdbab5: 2023-07-22T12:38:48,019 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1690029431661.4d15624c81ab8d3881a6ef2c42fdbab5. 2023-07-22T12:38:48,019 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing e1879fe3670023eba374a8cb2f79a944, disabling compactions & flushes 2023-07-22T12:38:48,019 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1690029417642.e1879fe3670023eba374a8cb2f79a944. 2023-07-22T12:38:48,019 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1690029417642.e1879fe3670023eba374a8cb2f79a944. 2023-07-22T12:38:48,019 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1690029417642.e1879fe3670023eba374a8cb2f79a944. after waiting 0 ms 2023-07-22T12:38:48,019 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1690029417642.e1879fe3670023eba374a8cb2f79a944. 2023-07-22T12:38:48,020 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing e1879fe3670023eba374a8cb2f79a944 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-07-22T12:38:48,021 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/hbase/meta/1588230740/.tmp/table/884d03f5a0074424a3fa0df8ab6b04c6 is 59, key is ns2:test-16900294235331/table:state/1690029427085/Put/seqid=0 2023-07-22T12:38:48,037 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/hbase/namespace/e1879fe3670023eba374a8cb2f79a944/.tmp/info/37e537d25c244f54a3ef0339b5783e94 is 45, key is default/info:d/1690029419019/Put/seqid=0 2023-07-22T12:38:48,143 DEBUG [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, b8af53427af1873ee686b4b22616f51c, dce7657497bf6b1da7900abd4083b1b8, e1879fe3670023eba374a8cb2f79a944 2023-07-22T12:38:48,159 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-07-22T12:38:48,159 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-07-22T12:38:48,344 DEBUG [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, b8af53427af1873ee686b4b22616f51c, dce7657497bf6b1da7900abd4083b1b8, e1879fe3670023eba374a8cb2f79a944 2023-07-22T12:38:48,431 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=714 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/hbase/meta/1588230740/.tmp/table/884d03f5a0074424a3fa0df8ab6b04c6 2023-07-22T12:38:48,436 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 884d03f5a0074424a3fa0df8ab6b04c6 2023-07-22T12:38:48,437 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/hbase/meta/1588230740/.tmp/info/a8225522ff66486aa8f3ccd91b399aab as hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/hbase/meta/1588230740/info/a8225522ff66486aa8f3ccd91b399aab 2023-07-22T12:38:48,442 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/hbase/meta/1588230740/info/a8225522ff66486aa8f3ccd91b399aab, entries=70, sequenceid=39, filesize=14.1 K 2023-07-22T12:38:48,443 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/hbase/meta/1588230740/.tmp/table/884d03f5a0074424a3fa0df8ab6b04c6 as hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/hbase/meta/1588230740/table/884d03f5a0074424a3fa0df8ab6b04c6 2023-07-22T12:38:48,447 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/hbase/meta/1588230740/table/884d03f5a0074424a3fa0df8ab6b04c6, entries=14, sequenceid=39, filesize=5.8 K 2023-07-22T12:38:48,448 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/hbase/namespace/e1879fe3670023eba374a8cb2f79a944/.tmp/info/37e537d25c244f54a3ef0339b5783e94 2023-07-22T12:38:48,448 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~9.25 KB/9474, heapSize ~15.92 KB/16304, currentSize=0 B/0 for 1588230740 in 505ms, sequenceid=39, compaction requested=false 2023-07-22T12:38:48,451 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a8225522ff66486aa8f3ccd91b399aab 2023-07-22T12:38:48,453 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 884d03f5a0074424a3fa0df8ab6b04c6 2023-07-22T12:38:48,458 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/hbase/meta/1588230740/recovered.edits/42.seqid, newMaxSeqId=42, maxSeqId=1 2023-07-22T12:38:48,458 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 37e537d25c244f54a3ef0339b5783e94 2023-07-22T12:38:48,458 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:38:48,458 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-07-22T12:38:48,459 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-07-22T12:38:48,459 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-07-22T12:38:48,459 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-07-22T12:38:48,459 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/hbase/namespace/e1879fe3670023eba374a8cb2f79a944/.tmp/info/37e537d25c244f54a3ef0339b5783e94 as hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/hbase/namespace/e1879fe3670023eba374a8cb2f79a944/info/37e537d25c244f54a3ef0339b5783e94 2023-07-22T12:38:48,464 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/hbase/namespace/e1879fe3670023eba374a8cb2f79a944/info/37e537d25c244f54a3ef0339b5783e94, entries=7, sequenceid=11, filesize=5.1 K 2023-07-22T12:38:48,465 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for e1879fe3670023eba374a8cb2f79a944 in 446ms, sequenceid=11, compaction requested=false 2023-07-22T12:38:48,467 DEBUG [StoreFileCloser-e1879fe3670023eba374a8cb2f79a944-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 37e537d25c244f54a3ef0339b5783e94 2023-07-22T12:38:48,470 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/hbase/namespace/e1879fe3670023eba374a8cb2f79a944/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-07-22T12:38:48,471 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:38:48,471 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1690029417642.e1879fe3670023eba374a8cb2f79a944. 2023-07-22T12:38:48,471 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for e1879fe3670023eba374a8cb2f79a944: 2023-07-22T12:38:48,472 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1690029417642.e1879fe3670023eba374a8cb2f79a944. 2023-07-22T12:38:48,472 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing b8af53427af1873ee686b4b22616f51c, disabling compactions & flushes 2023-07-22T12:38:48,472 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. 2023-07-22T12:38:48,472 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. 2023-07-22T12:38:48,472 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. after waiting 0 ms 2023-07-22T12:38:48,472 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. 2023-07-22T12:38:48,472 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing b8af53427af1873ee686b4b22616f51c 2/2 column families, dataSize=1.93 KB heapSize=3.31 KB 2023-07-22T12:38:48,476 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/meta/5a8bac356821480db610738007903dc5 is 96, key is incrbackupset:hdfs://localhost:33313/backupUT/meta:ns2:test-16900294235331/1690029527845/Put/seqid=0 2023-07-22T12:38:48,544 DEBUG [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(1609): Waiting on b8af53427af1873ee686b4b22616f51c, dce7657497bf6b1da7900abd4083b1b8 2023-07-22T12:38:48,744 DEBUG [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(1609): Waiting on b8af53427af1873ee686b4b22616f51c, dce7657497bf6b1da7900abd4083b1b8 2023-07-22T12:38:48,881 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=308 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/meta/5a8bac356821480db610738007903dc5 2023-07-22T12:38:48,886 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5a8bac356821480db610738007903dc5 2023-07-22T12:38:48,886 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 5a8bac356821480db610738007903dc5 2023-07-22T12:38:48,889 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/session/3266f73be3d54221acec0b0257820926 is 398, key is session:backup_1690029510908/session:context/1690029527820/Put/seqid=0 2023-07-22T12:38:48,944 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(1579): Waiting on 2 regions to close 2023-07-22T12:38:48,944 DEBUG [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(1583): Online Regions={b8af53427af1873ee686b4b22616f51c=backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c., dce7657497bf6b1da7900abd4083b1b8=test-1690029423533,,1690029424029.dce7657497bf6b1da7900abd4083b1b8.} 2023-07-22T12:38:48,945 DEBUG [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(1609): Waiting on b8af53427af1873ee686b4b22616f51c, dce7657497bf6b1da7900abd4083b1b8 2023-07-22T12:38:49,145 DEBUG [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(1609): Waiting on b8af53427af1873ee686b4b22616f51c, dce7657497bf6b1da7900abd4083b1b8 2023-07-22T12:38:49,294 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.63 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/session/3266f73be3d54221acec0b0257820926 2023-07-22T12:38:49,299 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3266f73be3d54221acec0b0257820926 2023-07-22T12:38:49,299 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3266f73be3d54221acec0b0257820926 2023-07-22T12:38:49,300 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/meta/5a8bac356821480db610738007903dc5 as hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/5a8bac356821480db610738007903dc5 2023-07-22T12:38:49,304 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5a8bac356821480db610738007903dc5 2023-07-22T12:38:49,304 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/meta/5a8bac356821480db610738007903dc5, entries=4, sequenceid=97, filesize=5.4 K 2023-07-22T12:38:49,305 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/.tmp/session/3266f73be3d54221acec0b0257820926 as hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/3266f73be3d54221acec0b0257820926 2023-07-22T12:38:49,309 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3266f73be3d54221acec0b0257820926 2023-07-22T12:38:49,309 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/session/3266f73be3d54221acec0b0257820926, entries=5, sequenceid=97, filesize=5.7 K 2023-07-22T12:38:49,310 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~1.93 KB/1981, heapSize ~3.28 KB/3360, currentSize=0 B/0 for b8af53427af1873ee686b4b22616f51c in 838ms, sequenceid=97, compaction requested=true 2023-07-22T12:38:49,312 DEBUG [StoreFileCloser-b8af53427af1873ee686b4b22616f51c-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8b4e5904fd5541caabafed2e0a7bbecc 2023-07-22T12:38:49,313 DEBUG [StoreFileCloser-b8af53427af1873ee686b4b22616f51c-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: d7524c21e0884ef7a726165d97bdf81e 2023-07-22T12:38:49,315 DEBUG [StoreFileCloser-b8af53427af1873ee686b4b22616f51c-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 4b24f1898bb44138bbfed9c444ed2091 2023-07-22T12:38:49,316 DEBUG [StoreFileCloser-b8af53427af1873ee686b4b22616f51c-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8f72b04c78fb4f5f8c4135a5820932d6 2023-07-22T12:38:49,317 DEBUG [StoreFileCloser-b8af53427af1873ee686b4b22616f51c-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: f1b01e23079e4b62937f7c002d957751 2023-07-22T12:38:49,319 DEBUG [StoreFileCloser-b8af53427af1873ee686b4b22616f51c-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 44d90b27cb2646e2a8f344c0895cc914 2023-07-22T12:38:49,320 DEBUG [StoreFileCloser-b8af53427af1873ee686b4b22616f51c-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 9e6a4ccee5334e62bda59100f05e6b43 2023-07-22T12:38:49,321 DEBUG [StoreFileCloser-b8af53427af1873ee686b4b22616f51c-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 5a8bac356821480db610738007903dc5 2023-07-22T12:38:49,323 DEBUG [StoreFileCloser-b8af53427af1873ee686b4b22616f51c-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e7980d1347624a0e8cf8cbac22176c14 2023-07-22T12:38:49,324 DEBUG [StoreFileCloser-b8af53427af1873ee686b4b22616f51c-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 40fe5705ee804503bdd7677bd59e3be8 2023-07-22T12:38:49,325 DEBUG [StoreFileCloser-b8af53427af1873ee686b4b22616f51c-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 882f60fdcf9248459ac9b39b6e8e1f01 2023-07-22T12:38:49,327 DEBUG [StoreFileCloser-b8af53427af1873ee686b4b22616f51c-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 43bbdf59d84f4c98ac7bd38ac51d7e69 2023-07-22T12:38:49,328 DEBUG [StoreFileCloser-b8af53427af1873ee686b4b22616f51c-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a30af91d0ecb4b06a2e7606fcf805b33 2023-07-22T12:38:49,329 DEBUG [StoreFileCloser-b8af53427af1873ee686b4b22616f51c-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e19bff70c6164f2581f82c4faaec10d1 2023-07-22T12:38:49,332 DEBUG [StoreFileCloser-b8af53427af1873ee686b4b22616f51c-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 27212316e8204a85bde5b84e3647aff0 2023-07-22T12:38:49,336 DEBUG [StoreFileCloser-b8af53427af1873ee686b4b22616f51c-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: d2eaeb75a51f492893c7020dc1cbb5c5 2023-07-22T12:38:49,336 DEBUG [StoreFileCloser-b8af53427af1873ee686b4b22616f51c-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3266f73be3d54221acec0b0257820926 2023-07-22T12:38:49,339 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/backup/system/b8af53427af1873ee686b4b22616f51c/recovered.edits/100.seqid, newMaxSeqId=100, maxSeqId=1 2023-07-22T12:38:49,340 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:38:49,340 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. 2023-07-22T12:38:49,340 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for b8af53427af1873ee686b4b22616f51c: 2023-07-22T12:38:49,340 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1690029431032.b8af53427af1873ee686b4b22616f51c. 2023-07-22T12:38:49,340 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing dce7657497bf6b1da7900abd4083b1b8, disabling compactions & flushes 2023-07-22T12:38:49,340 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1690029423533,,1690029424029.dce7657497bf6b1da7900abd4083b1b8. 2023-07-22T12:38:49,340 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1690029423533,,1690029424029.dce7657497bf6b1da7900abd4083b1b8. 2023-07-22T12:38:49,340 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1690029423533,,1690029424029.dce7657497bf6b1da7900abd4083b1b8. after waiting 0 ms 2023-07-22T12:38:49,340 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1690029423533,,1690029424029.dce7657497bf6b1da7900abd4083b1b8. 2023-07-22T12:38:49,341 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing dce7657497bf6b1da7900abd4083b1b8 1/1 column families, dataSize=7.13 KB heapSize=21.91 KB 2023-07-22T12:38:49,344 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/default/test-1690029423533/dce7657497bf6b1da7900abd4083b1b8/.tmp/f/e2398f3f081d40c882285cd02c48d60c is 42, key is row-t1100/f:q1/1690029476358/Put/seqid=0 2023-07-22T12:38:49,345 DEBUG [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(1609): Waiting on dce7657497bf6b1da7900abd4083b1b8 2023-07-22T12:38:49,545 DEBUG [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(1609): Waiting on dce7657497bf6b1da7900abd4083b1b8 2023-07-22T12:38:49,745 DEBUG [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(1609): Waiting on dce7657497bf6b1da7900abd4083b1b8 2023-07-22T12:38:49,748 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.13 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/default/test-1690029423533/dce7657497bf6b1da7900abd4083b1b8/.tmp/f/e2398f3f081d40c882285cd02c48d60c 2023-07-22T12:38:49,756 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e2398f3f081d40c882285cd02c48d60c 2023-07-22T12:38:49,757 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/default/test-1690029423533/dce7657497bf6b1da7900abd4083b1b8/.tmp/f/e2398f3f081d40c882285cd02c48d60c as hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/default/test-1690029423533/dce7657497bf6b1da7900abd4083b1b8/f/e2398f3f081d40c882285cd02c48d60c 2023-07-22T12:38:49,763 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/default/test-1690029423533/dce7657497bf6b1da7900abd4083b1b8/f/e2398f3f081d40c882285cd02c48d60c, entries=198, sequenceid=304, filesize=12.6 K 2023-07-22T12:38:49,763 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~7.13 KB/7304, heapSize ~21.89 KB/22416, currentSize=0 B/0 for dce7657497bf6b1da7900abd4083b1b8 in 423ms, sequenceid=304, compaction requested=false 2023-07-22T12:38:49,765 DEBUG [StoreFileCloser-dce7657497bf6b1da7900abd4083b1b8-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: b1910e2882a7483ea0a1c393d00f3b3a 2023-07-22T12:38:49,766 DEBUG [StoreFileCloser-dce7657497bf6b1da7900abd4083b1b8-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e2398f3f081d40c882285cd02c48d60c 2023-07-22T12:38:49,770 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/data/default/test-1690029423533/dce7657497bf6b1da7900abd4083b1b8/recovered.edits/307.seqid, newMaxSeqId=307, maxSeqId=1 2023-07-22T12:38:49,771 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:38:49,772 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1690029423533,,1690029424029.dce7657497bf6b1da7900abd4083b1b8. 2023-07-22T12:38:49,772 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for dce7657497bf6b1da7900abd4083b1b8: 2023-07-22T12:38:49,772 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1690029423533,,1690029424029.dce7657497bf6b1da7900abd4083b1b8. 2023-07-22T12:38:49,946 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,39283,1690029414077; all regions closed. 2023-07-22T12:38:49,953 DEBUG [RS:0;jenkins-hbase3:39283 {}] wal.AbstractFSWAL(1062): Moved 2 WAL file(s) to /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs 2023-07-22T12:38:49,953 INFO [RS:0;jenkins-hbase3:39283 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C39283%2C1690029414077.meta:.meta(num 1690029515111) 2023-07-22T12:38:49,962 DEBUG [RS:0;jenkins-hbase3:39283 {}] wal.AbstractFSWAL(1062): Moved 7 WAL file(s) to /user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/oldWALs 2023-07-22T12:38:49,962 INFO [RS:0;jenkins-hbase3:39283 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C39283%2C1690029414077:(num 1690029515088) 2023-07-22T12:38:49,962 DEBUG [RS:0;jenkins-hbase3:39283 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:38:49,962 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.LeaseManager(133): Closed leases 2023-07-22T12:38:49,962 INFO [RS:0;jenkins-hbase3:39283 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2023-07-22T12:38:49,963 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-07-22T12:38:49,964 INFO [RS:0;jenkins-hbase3:39283 {}] ipc.NettyRpcServer(213): Stopping server on /172.31.12.81:39283 2023-07-22T12:38:49,970 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-07-22T12:38:49,970 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rs/jenkins-hbase3.apache.org,39283,1690029414077 2023-07-22T12:38:49,973 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,39283,1690029414077] 2023-07-22T12:38:49,973 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,39283,1690029414077; numProcessing=1 2023-07-22T12:38:49,974 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(224): Node /1/draining/jenkins-hbase3.apache.org,39283,1690029414077 already deleted, retry=false 2023-07-22T12:38:49,974 INFO [RegionServerTracker-0 {}] master.ServerManager(569): Cluster shutdown set; jenkins-hbase3.apache.org,39283,1690029414077 expired; onlineServers=0 2023-07-22T12:38:49,974 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2513): ***** STOPPING region server 'jenkins-hbase3.apache.org,33617,1690029410929' ***** 2023-07-22T12:38:49,974 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2527): STOPPED: Cluster shutdown set; onlineServer=0 2023-07-22T12:38:49,975 DEBUG [M:0;jenkins-hbase3:33617 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6856295d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-07-22T12:38:49,975 INFO [M:0;jenkins-hbase3:33617 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-07-22T12:38:49,981 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/master 2023-07-22T12:38:49,981 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:38:49,982 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-07-22T12:38:50,019 INFO [M:0;jenkins-hbase3:33617 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@3552251f{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-07-22T12:38:50,019 INFO [M:0;jenkins-hbase3:33617 {}] server.AbstractConnector(383): Stopped ServerConnector@644ab3cd{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-07-22T12:38:50,020 INFO [M:0;jenkins-hbase3:33617 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-07-22T12:38:50,020 INFO [M:0;jenkins-hbase3:33617 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@127c31ba{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-07-22T12:38:50,021 INFO [M:0;jenkins-hbase3:33617 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@302eec96{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/hadoop.log.dir/,STOPPED} 2023-07-22T12:38:50,021 INFO [M:0;jenkins-hbase3:33617 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,33617,1690029410929 2023-07-22T12:38:50,021 INFO [M:0;jenkins-hbase3:33617 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,33617,1690029410929; all regions closed. 2023-07-22T12:38:50,021 DEBUG [M:0;jenkins-hbase3:33617 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:38:50,021 INFO [M:0;jenkins-hbase3:33617 {}] master.HMaster(1664): Stopping master jetty server 2023-07-22T12:38:50,022 INFO [M:0;jenkins-hbase3:33617 {}] server.AbstractConnector(383): Stopped ServerConnector@5d154786{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-07-22T12:38:50,022 DEBUG [M:0;jenkins-hbase3:33617 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-07-22T12:38:50,022 INFO [M:0;jenkins-hbase3:33617 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-07-22T12:38:50,022 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-07-22T12:38:50,023 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1690029415964 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1690029415964,5,FailOnTimeoutGroup] 2023-07-22T12:38:50,023 DEBUG [M:0;jenkins-hbase3:33617 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1ebb090b to 127.0.0.1:51445 2023-07-22T12:38:50,023 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1690029415964 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1690029415964,5,FailOnTimeoutGroup] 2023-07-22T12:38:50,023 DEBUG [M:0;jenkins-hbase3:33617 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:38:50,023 DEBUG [M:0;jenkins-hbase3:33617 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-07-22T12:38:50,023 INFO [M:0;jenkins-hbase3:33617 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-07-22T12:38:50,023 DEBUG [M:0;jenkins-hbase3:33617 {}] master.HMaster(1687): Stopping service threads 2023-07-22T12:38:50,023 INFO [M:0;jenkins-hbase3:33617 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-07-22T12:38:50,023 ERROR [M:0;jenkins-hbase3:33617 {}] procedure2.ProcedureExecutor(655): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] 2023-07-22T12:38:50,024 INFO [M:0;jenkins-hbase3:33617 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-07-22T12:38:50,024 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-07-22T12:38:50,024 DEBUG [M:0;jenkins-hbase3:33617 {}] zookeeper.ZKUtil(347): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Unable to get data of znode /1/master because node does not exist (not an error) 2023-07-22T12:38:50,024 WARN [M:0;jenkins-hbase3:33617 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-07-22T12:38:50,024 INFO [M:0;jenkins-hbase3:33617 {}] assignment.AssignmentManager(382): Stopping assignment manager 2023-07-22T12:38:50,025 INFO [M:0;jenkins-hbase3:33617 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-07-22T12:38:50,025 DEBUG [M:0;jenkins-hbase3:33617 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-07-22T12:38:50,036 INFO [M:0;jenkins-hbase3:33617 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:38:50,036 DEBUG [M:0;jenkins-hbase3:33617 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:38:50,036 DEBUG [M:0;jenkins-hbase3:33617 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-07-22T12:38:50,036 DEBUG [M:0;jenkins-hbase3:33617 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:38:50,036 INFO [M:0;jenkins-hbase3:33617 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=201.94 KB heapSize=244.75 KB 2023-07-22T12:38:50,050 DEBUG [M:0;jenkins-hbase3:33617 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/15e4c5dc6d494d53a3c567a1a27e9dd9 is 88, key is hbase:meta,,1/info:sn/1690029416966/Put/seqid=0 2023-07-22T12:38:50,054 INFO [M:0;jenkins-hbase3:33617 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/15e4c5dc6d494d53a3c567a1a27e9dd9 2023-07-22T12:38:50,060 DEBUG [M:0;jenkins-hbase3:33617 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 15e4c5dc6d494d53a3c567a1a27e9dd9 2023-07-22T12:38:50,072 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-07-22T12:38:50,072 INFO [RS:0;jenkins-hbase3:39283 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,39283,1690029414077; zookeeper connection closed. 2023-07-22T12:38:50,072 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): regionserver:39283-0x1000063a7240001, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-07-22T12:38:50,073 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@bf68aaa {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@bf68aaa 2023-07-22T12:38:50,073 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-07-22T12:38:50,075 DEBUG [M:0;jenkins-hbase3:33617 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/335a589eb6274a06882cb25508b7dea8 is 986, key is \x00\x00\x00\x00\x00\x00\x00\x1D/proc:d/1690029432862/Put/seqid=0 2023-07-22T12:38:50,081 INFO [M:0;jenkins-hbase3:33617 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=201.35 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/335a589eb6274a06882cb25508b7dea8 2023-07-22T12:38:50,086 INFO [M:0;jenkins-hbase3:33617 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 335a589eb6274a06882cb25508b7dea8 2023-07-22T12:38:50,086 DEBUG [M:0;jenkins-hbase3:33617 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 335a589eb6274a06882cb25508b7dea8 2023-07-22T12:38:50,098 DEBUG [M:0;jenkins-hbase3:33617 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3841697ca3eb43b5a30598a55ddcdb7e is 82, key is jenkins-hbase3.apache.org,39283,1690029414077/rs:state/1690029416071/Put/seqid=0 2023-07-22T12:38:50,502 INFO [M:0;jenkins-hbase3:33617 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3841697ca3eb43b5a30598a55ddcdb7e 2023-07-22T12:38:50,507 DEBUG [M:0;jenkins-hbase3:33617 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3841697ca3eb43b5a30598a55ddcdb7e 2023-07-22T12:38:50,508 DEBUG [M:0;jenkins-hbase3:33617 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/15e4c5dc6d494d53a3c567a1a27e9dd9 as hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/15e4c5dc6d494d53a3c567a1a27e9dd9 2023-07-22T12:38:50,512 INFO [M:0;jenkins-hbase3:33617 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/15e4c5dc6d494d53a3c567a1a27e9dd9, entries=8, sequenceid=507, filesize=5.5 K 2023-07-22T12:38:50,513 DEBUG [M:0;jenkins-hbase3:33617 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/335a589eb6274a06882cb25508b7dea8 as hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/335a589eb6274a06882cb25508b7dea8 2023-07-22T12:38:50,517 INFO [M:0;jenkins-hbase3:33617 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 335a589eb6274a06882cb25508b7dea8 2023-07-22T12:38:50,517 INFO [M:0;jenkins-hbase3:33617 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/335a589eb6274a06882cb25508b7dea8, entries=53, sequenceid=507, filesize=16.6 K 2023-07-22T12:38:50,518 DEBUG [M:0;jenkins-hbase3:33617 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3841697ca3eb43b5a30598a55ddcdb7e as hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3841697ca3eb43b5a30598a55ddcdb7e 2023-07-22T12:38:50,522 INFO [M:0;jenkins-hbase3:33617 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33313/user/jenkins/test-data/6a971ac3-dd57-0ab5-ac41-56237be09a2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3841697ca3eb43b5a30598a55ddcdb7e, entries=1, sequenceid=507, filesize=5.1 K 2023-07-22T12:38:50,523 INFO [M:0;jenkins-hbase3:33617 {}] regionserver.HRegion(3022): Finished flush of dataSize ~201.94 KB/206789, heapSize ~244.45 KB/250320, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 487ms, sequenceid=507, compaction requested=false 2023-07-22T12:38:50,524 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 15e4c5dc6d494d53a3c567a1a27e9dd9 2023-07-22T12:38:50,526 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-proc-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 335a589eb6274a06882cb25508b7dea8 2023-07-22T12:38:50,527 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-rs-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3841697ca3eb43b5a30598a55ddcdb7e 2023-07-22T12:38:50,527 INFO [M:0;jenkins-hbase3:33617 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:38:50,528 DEBUG [M:0;jenkins-hbase3:33617 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-07-22T12:38:50,530 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-07-22T12:38:50,530 INFO [M:0;jenkins-hbase3:33617 {}] flush.MasterFlushTableProcedureManager(81): stop: server shutting down. 2023-07-22T12:38:50,530 INFO [M:0;jenkins-hbase3:33617 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-07-22T12:38:50,531 INFO [M:0;jenkins-hbase3:33617 {}] ipc.NettyRpcServer(213): Stopping server on /172.31.12.81:33617 2023-07-22T12:38:50,533 DEBUG [M:0;jenkins-hbase3:33617 {}] zookeeper.RecoverableZooKeeper(224): Node /1/rs/jenkins-hbase3.apache.org,33617,1690029410929 already deleted, retry=false 2023-07-22T12:38:50,635 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-07-22T12:38:50,635 INFO [M:0;jenkins-hbase3:33617 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,33617,1690029410929; zookeeper connection closed. 2023-07-22T12:38:50,635 DEBUG [Listener at localhost/36365-EventThread {}] zookeeper.ZKWatcher(604): master:33617-0x1000063a7240000, quorum=127.0.0.1:51445, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-07-22T12:38:50,641 WARN [Listener at jenkins-hbase3.apache.org/34965 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-07-22T12:38:50,644 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-07-22T12:38:50,753 WARN [BP-1667261202-172.31.12.81-1690029407680 heartbeating to localhost/127.0.0.1:33313 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-07-22T12:38:50,754 WARN [BP-1667261202-172.31.12.81-1690029407680 heartbeating to localhost/127.0.0.1:33313 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-1667261202-172.31.12.81-1690029407680 (Datanode Uuid fee69a25-9af2-43eb-8040-db30620d1a9d) service to localhost/127.0.0.1:33313 2023-07-22T12:38:50,756 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/cluster_a7b0bb94-85a8-a89c-f1a4-3863f52e4e30/dfs/data/data1/current/BP-1667261202-172.31.12.81-1690029407680 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-07-22T12:38:50,756 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/156cfa6d-5f17-f4d7-443f-f70429fc1bef/cluster_a7b0bb94-85a8-a89c-f1a4-3863f52e4e30/dfs/data/data2/current/BP-1667261202-172.31.12.81-1690029407680 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-07-22T12:38:50,790 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-07-22T12:38:50,908 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2023-07-22T12:38:50,976 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-07-22T12:38:50,976 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-07-22T12:38:50,987 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-07-22T12:38:51,094 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-07-22T12:38:51,094 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-07-22T12:38:51,096 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-07-22T12:38:51,097 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16900294235332 2023-07-22T12:38:51,098 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-07-22T12:38:51,099 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-07-22T12:38:51,104 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-07-22T12:38:51,105 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16900294235333 2023-07-22T12:38:51,105 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1690029423533 2023-07-22T12:38:51,107 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-07-22T12:38:51,108 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16900294235331 2023-07-22T12:38:51,109 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:38:51,221 ERROR [Thread[Thread-275,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-07-22T12:38:51,221 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-07-22T12:38:51,329 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-07-22T12:38:51,336 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-07-22T12:38:51,337 ERROR [Thread[Thread-285,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-07-22T12:38:51,343 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-07-22T12:38:51,446 ERROR [Thread[Thread-250,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-07-22T12:38:51,446 INFO [Listener at jenkins-hbase3.apache.org/34965 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped