2023-09-06 20:33:45,680 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7c7b252e 2023-09-06 20:33:45,694 main DEBUG Took 0.011132 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-09-06 20:33:45,694 main DEBUG PluginManager 'Core' found 129 plugins 2023-09-06 20:33:45,694 main DEBUG PluginManager 'Level' found 0 plugins 2023-09-06 20:33:45,695 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-09-06 20:33:45,697 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:33:45,705 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-09-06 20:33:45,717 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:33:45,719 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:33:45,720 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:33:45,720 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:33:45,720 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:33:45,720 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:33:45,721 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:33:45,721 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:33:45,722 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:33:45,722 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:33:45,723 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:33:45,723 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:33:45,723 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:33:45,724 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:33:45,724 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:33:45,724 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:33:45,724 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:33:45,725 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:33:45,725 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:33:45,725 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:33:45,726 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:33:45,726 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:33:45,726 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:33:45,727 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:33:45,727 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:33:45,727 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-09-06 20:33:45,729 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:33:45,731 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-09-06 20:33:45,732 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-09-06 20:33:45,733 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-09-06 20:33:45,734 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-09-06 20:33:45,734 main DEBUG PluginManager 'Converter' found 47 plugins 2023-09-06 20:33:45,746 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-09-06 20:33:45,748 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-09-06 20:33:45,750 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-09-06 20:33:45,750 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-09-06 20:33:45,750 main DEBUG createAppenders(={Console}) 2023-09-06 20:33:45,751 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7c7b252e initialized 2023-09-06 20:33:45,751 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7c7b252e 2023-09-06 20:33:45,752 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7c7b252e OK. 2023-09-06 20:33:45,752 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-09-06 20:33:45,752 main DEBUG OutputStream closed 2023-09-06 20:33:45,752 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-09-06 20:33:45,753 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-09-06 20:33:45,753 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@72a7c7e0 OK 2023-09-06 20:33:45,795 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-09-06 20:33:45,797 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-09-06 20:33:45,798 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-09-06 20:33:45,799 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-09-06 20:33:45,800 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-09-06 20:33:45,800 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-09-06 20:33:45,800 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-09-06 20:33:45,801 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-09-06 20:33:45,801 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-09-06 20:33:45,801 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-09-06 20:33:45,801 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-09-06 20:33:45,802 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-09-06 20:33:45,802 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-09-06 20:33:45,802 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-09-06 20:33:45,803 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-09-06 20:33:45,803 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-09-06 20:33:45,803 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-09-06 20:33:45,804 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-09-06 20:33:45,807 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-09-06 20:33:45,807 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@e7edb54) with optional ClassLoader: null 2023-09-06 20:33:45,807 main DEBUG Shutdown hook enabled. Registering a new one. 2023-09-06 20:33:45,808 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@e7edb54] started OK. 2023-09-06T20:33:45,840 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures timeout: 13 mins 2023-09-06 20:33:45,844 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-09-06 20:33:45,844 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-09-06T20:33:46,176 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e 2023-09-06T20:33:46,181 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-09-06T20:33:46,183 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:33:46,184 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-09-06T20:33:46,188 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/cluster_17959e2a-7692-c4d5-e8b6-626f5ab10d8e, deleteOnExit=true 2023-09-06T20:33:46,188 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-09-06T20:33:46,188 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/test.cache.data in system properties and HBase conf 2023-09-06T20:33:46,189 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/hadoop.tmp.dir in system properties and HBase conf 2023-09-06T20:33:46,189 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/hadoop.log.dir in system properties and HBase conf 2023-09-06T20:33:46,190 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/mapreduce.cluster.local.dir in system properties and HBase conf 2023-09-06T20:33:46,190 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-09-06T20:33:46,190 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-09-06T20:33:46,303 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-09-06T20:33:46,734 DEBUG [Time-limited test {}] fs.HFileSystem(308): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-09-06T20:33:46,739 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-09-06T20:33:46,739 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-09-06T20:33:46,740 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-09-06T20:33:46,740 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-06T20:33:46,740 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-09-06T20:33:46,741 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-09-06T20:33:46,741 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-06T20:33:46,741 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-06T20:33:46,741 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-09-06T20:33:46,742 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/nfs.dump.dir in system properties and HBase conf 2023-09-06T20:33:46,742 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/java.io.tmpdir in system properties and HBase conf 2023-09-06T20:33:46,742 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-06T20:33:46,743 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-09-06T20:33:46,743 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-09-06T20:33:47,212 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-09-06T20:33:47,216 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-09-06T20:33:47,505 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-09-06T20:33:47,646 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@1976f7d5 via org.mortbay.log.Slf4jLog 2023-09-06T20:33:47,662 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:33:47,703 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:33:47,741 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/java.io.tmpdir/Jetty_localhost_44525_hdfs____wnyuzm/webapp 2023-09-06T20:33:47,874 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:44525 2023-09-06T20:33:47,901 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-09-06T20:33:47,901 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-09-06T20:33:48,436 WARN [Listener at localhost/38135 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-09-06T20:33:48,498 WARN [Listener at localhost/38135 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-09-06T20:33:48,516 WARN [Listener at localhost/38135 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:33:48,522 INFO [Listener at localhost/38135 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:33:48,528 INFO [Listener at localhost/38135 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/java.io.tmpdir/Jetty_localhost_34655_datanode____.uiphz8/webapp 2023-09-06T20:33:48,627 INFO [Listener at localhost/38135 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:34655 2023-09-06T20:33:48,922 WARN [Listener at localhost/46585 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-09-06T20:33:49,212 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-09-06T20:33:49,392 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xe49ff4f69d2fc1fb: Processing first storage report for DS-edff5bfa-143b-471c-9d7d-03661acff561 from datanode cedb6886-cf3f-4a99-b36b-a59b3394261c 2023-09-06T20:33:49,394 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xe49ff4f69d2fc1fb: from storage DS-edff5bfa-143b-471c-9d7d-03661acff561 node DatanodeRegistration(127.0.0.1:37183, datanodeUuid=cedb6886-cf3f-4a99-b36b-a59b3394261c, infoPort=34309, infoSecurePort=0, ipcPort=46585, storageInfo=lv=-57;cid=testClusterID;nsid=1172971742;c=1694032427287), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2023-09-06T20:33:49,394 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xe49ff4f69d2fc1fb: Processing first storage report for DS-db7ea517-9d79-4794-8baf-cb35023c50db from datanode cedb6886-cf3f-4a99-b36b-a59b3394261c 2023-09-06T20:33:49,394 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xe49ff4f69d2fc1fb: from storage DS-db7ea517-9d79-4794-8baf-cb35023c50db node DatanodeRegistration(127.0.0.1:37183, datanodeUuid=cedb6886-cf3f-4a99-b36b-a59b3394261c, infoPort=34309, infoSecurePort=0, ipcPort=46585, storageInfo=lv=-57;cid=testClusterID;nsid=1172971742;c=1694032427287), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-09-06T20:33:49,441 DEBUG [Listener at localhost/46585 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e 2023-09-06T20:33:49,542 INFO [Listener at localhost/46585 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/cluster_17959e2a-7692-c4d5-e8b6-626f5ab10d8e/zookeeper_0, clientPort=51523, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/cluster_17959e2a-7692-c4d5-e8b6-626f5ab10d8e/zookeeper_0/version-2, dataDirSize=424 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/cluster_17959e2a-7692-c4d5-e8b6-626f5ab10d8e/zookeeper_0/version-2, dataLogSize=424 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, serverId=0 2023-09-06T20:33:49,560 INFO [Listener at localhost/46585 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=51523 2023-09-06T20:33:49,568 INFO [Listener at localhost/46585 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:33:49,570 INFO [Listener at localhost/46585 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:33:50,234 INFO [Listener at localhost/46585 {}] util.FSUtils(461): Created version file at hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928 with version=8 2023-09-06T20:33:50,234 INFO [Listener at localhost/46585 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/hbase-staging 2023-09-06T20:33:50,251 DEBUG [Listener at localhost/46585 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-09-06T20:33:50,252 DEBUG [Listener at localhost/46585 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-09-06T20:33:50,252 DEBUG [Listener at localhost/46585 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-09-06T20:33:50,252 DEBUG [Listener at localhost/46585 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-09-06T20:33:50,470 DEBUG [Listener at localhost/46585 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-09-06T20:33:51,233 INFO [Listener at localhost/46585 {}] client.ConnectionUtils(128): master/jenkins-hbase3:0 server-side Connection retries=45 2023-09-06T20:33:51,281 INFO [Listener at localhost/46585 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-06T20:33:51,282 INFO [Listener at localhost/46585 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-09-06T20:33:51,282 INFO [Listener at localhost/46585 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-09-06T20:33:51,282 INFO [Listener at localhost/46585 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-06T20:33:51,283 INFO [Listener at localhost/46585 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-09-06T20:33:51,436 INFO [Listener at localhost/46585 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-09-06T20:33:51,565 INFO [Listener at localhost/46585 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-09-06T20:33:51,584 DEBUG [Listener at localhost/46585 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-09-06T20:33:51,590 INFO [Listener at localhost/46585 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-09-06T20:33:51,639 DEBUG [Listener at localhost/46585 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 32291 (auto-detected) 2023-09-06T20:33:51,641 DEBUG [Listener at localhost/46585 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:1c:ff:fe:e2:13:ea (auto-detected) 2023-09-06T20:33:51,679 INFO [Listener at localhost/46585 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:45185 2023-09-06T20:33:51,693 INFO [Listener at localhost/46585 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:33:51,697 INFO [Listener at localhost/46585 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:33:51,720 INFO [Listener at localhost/46585 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:45185 connecting to ZooKeeper ensemble=127.0.0.1:51523 2023-09-06T20:33:51,766 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:451850x0, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-09-06T20:33:51,770 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:45185-0x100eefce21b0000 connected 2023-09-06T20:33:51,801 DEBUG [Listener at localhost/46585 {}] zookeeper.ZKUtil(113): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-09-06T20:33:51,804 DEBUG [Listener at localhost/46585 {}] zookeeper.ZKUtil(113): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-09-06T20:33:51,808 DEBUG [Listener at localhost/46585 {}] zookeeper.ZKUtil(113): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-09-06T20:33:51,818 DEBUG [Listener at localhost/46585 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45185 2023-09-06T20:33:51,818 DEBUG [Listener at localhost/46585 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45185 2023-09-06T20:33:51,819 DEBUG [Listener at localhost/46585 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45185 2023-09-06T20:33:51,820 DEBUG [Listener at localhost/46585 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45185 2023-09-06T20:33:51,821 DEBUG [Listener at localhost/46585 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45185 2023-09-06T20:33:51,875 INFO [Listener at localhost/46585 {}] log.Log(170): Logging initialized @6823ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-09-06T20:33:52,009 INFO [Listener at localhost/46585 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-09-06T20:33:52,010 INFO [Listener at localhost/46585 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-09-06T20:33:52,011 INFO [Listener at localhost/46585 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-09-06T20:33:52,013 INFO [Listener at localhost/46585 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-09-06T20:33:52,014 INFO [Listener at localhost/46585 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-09-06T20:33:52,014 INFO [Listener at localhost/46585 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-09-06T20:33:52,017 INFO [Listener at localhost/46585 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-09-06T20:33:52,150 INFO [Listener at localhost/46585 {}] http.HttpServer(1219): Jetty bound to port 44971 2023-09-06T20:33:52,152 INFO [Listener at localhost/46585 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-09-06T20:33:52,183 INFO [Listener at localhost/46585 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:33:52,187 INFO [Listener at localhost/46585 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@55869636{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/hadoop.log.dir/,AVAILABLE} 2023-09-06T20:33:52,188 INFO [Listener at localhost/46585 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:33:52,188 INFO [Listener at localhost/46585 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@2c6b8bd5{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-09-06T20:33:52,256 INFO [Listener at localhost/46585 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-09-06T20:33:52,270 INFO [Listener at localhost/46585 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-09-06T20:33:52,270 INFO [Listener at localhost/46585 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-09-06T20:33:52,272 INFO [Listener at localhost/46585 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-09-06T20:33:52,280 INFO [Listener at localhost/46585 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:33:53,390 INFO [Listener at localhost/46585 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@5d1c5812{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-09-06T20:33:53,402 INFO [Listener at localhost/46585 {}] server.AbstractConnector(333): Started ServerConnector@452071dd{HTTP/1.1, (http/1.1)}{0.0.0.0:44971} 2023-09-06T20:33:53,402 INFO [Listener at localhost/46585 {}] server.Server(415): Started @8350ms 2023-09-06T20:33:53,406 INFO [Listener at localhost/46585 {}] master.HMaster(485): hbase.rootdir=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928, hbase.cluster.distributed=false 2023-09-06T20:33:53,467 INFO [Listener at localhost/46585 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase3:0 server-side Connection retries=45 2023-09-06T20:33:53,468 INFO [Listener at localhost/46585 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-06T20:33:53,468 INFO [Listener at localhost/46585 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-09-06T20:33:53,468 INFO [Listener at localhost/46585 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-09-06T20:33:53,468 INFO [Listener at localhost/46585 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-06T20:33:53,468 INFO [Listener at localhost/46585 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-09-06T20:33:53,476 INFO [Listener at localhost/46585 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-09-06T20:33:53,478 INFO [Listener at localhost/46585 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-09-06T20:33:53,480 INFO [Listener at localhost/46585 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:44489 2023-09-06T20:33:53,482 INFO [Listener at localhost/46585 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-09-06T20:33:53,490 DEBUG [Listener at localhost/46585 {}] mob.MobFileCache(120): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-09-06T20:33:53,491 INFO [Listener at localhost/46585 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:33:53,494 INFO [Listener at localhost/46585 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:33:53,496 INFO [Listener at localhost/46585 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:44489 connecting to ZooKeeper ensemble=127.0.0.1:51523 2023-09-06T20:33:53,499 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): regionserver:444890x0, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-09-06T20:33:53,500 DEBUG [Listener at localhost/46585 {}] zookeeper.ZKUtil(113): regionserver:444890x0, quorum=127.0.0.1:51523, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-09-06T20:33:53,500 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:44489-0x100eefce21b0001 connected 2023-09-06T20:33:53,502 DEBUG [Listener at localhost/46585 {}] zookeeper.ZKUtil(113): regionserver:44489-0x100eefce21b0001, quorum=127.0.0.1:51523, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-09-06T20:33:53,503 DEBUG [Listener at localhost/46585 {}] zookeeper.ZKUtil(113): regionserver:44489-0x100eefce21b0001, quorum=127.0.0.1:51523, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-09-06T20:33:53,503 DEBUG [Listener at localhost/46585 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44489 2023-09-06T20:33:53,503 DEBUG [Listener at localhost/46585 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44489 2023-09-06T20:33:53,504 DEBUG [Listener at localhost/46585 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44489 2023-09-06T20:33:53,504 DEBUG [Listener at localhost/46585 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44489 2023-09-06T20:33:53,505 DEBUG [Listener at localhost/46585 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44489 2023-09-06T20:33:53,509 INFO [Listener at localhost/46585 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-09-06T20:33:53,509 INFO [Listener at localhost/46585 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-09-06T20:33:53,509 INFO [Listener at localhost/46585 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-09-06T20:33:53,510 INFO [Listener at localhost/46585 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-09-06T20:33:53,510 INFO [Listener at localhost/46585 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-09-06T20:33:53,510 INFO [Listener at localhost/46585 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-09-06T20:33:53,511 INFO [Listener at localhost/46585 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-09-06T20:33:53,513 INFO [Listener at localhost/46585 {}] http.HttpServer(1219): Jetty bound to port 45649 2023-09-06T20:33:53,514 INFO [Listener at localhost/46585 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-09-06T20:33:53,515 INFO [Listener at localhost/46585 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:33:53,515 INFO [Listener at localhost/46585 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@679f1315{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/hadoop.log.dir/,AVAILABLE} 2023-09-06T20:33:53,516 INFO [Listener at localhost/46585 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:33:53,516 INFO [Listener at localhost/46585 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@1b920999{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-09-06T20:33:53,533 INFO [Listener at localhost/46585 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-09-06T20:33:53,534 INFO [Listener at localhost/46585 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-09-06T20:33:53,535 INFO [Listener at localhost/46585 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-09-06T20:33:53,535 INFO [Listener at localhost/46585 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-09-06T20:33:53,537 INFO [Listener at localhost/46585 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:33:53,541 INFO [Listener at localhost/46585 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@63f72d5b{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-09-06T20:33:53,542 INFO [Listener at localhost/46585 {}] server.AbstractConnector(333): Started ServerConnector@2e45869f{HTTP/1.1, (http/1.1)}{0.0.0.0:45649} 2023-09-06T20:33:53,543 INFO [Listener at localhost/46585 {}] server.Server(415): Started @8490ms 2023-09-06T20:33:53,548 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-09-06T20:33:53,551 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@302e9b29{HTTP/1.1, (http/1.1)}{0.0.0.0:46561} 2023-09-06T20:33:53,551 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @8499ms 2023-09-06T20:33:53,552 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2388): Adding backup master ZNode /1/backup-masters/jenkins-hbase3.apache.org,45185,1694032430454 2023-09-06T20:33:53,565 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44489-0x100eefce21b0001, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-09-06T20:33:53,565 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-09-06T20:33:53,567 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase3.apache.org,45185,1694032430454 2023-09-06T20:33:53,571 DEBUG [M:0;jenkins-hbase3:45185 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:45185 2023-09-06T20:33:53,586 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-09-06T20:33:53,586 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44489-0x100eefce21b0001, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-09-06T20:33:53,586 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44489-0x100eefce21b0001, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:33:53,586 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:33:53,587 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Set watcher on existing znode=/1/master 2023-09-06T20:33:53,589 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase3.apache.org,45185,1694032430454 from backup master directory 2023-09-06T20:33:53,589 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Set watcher on existing znode=/1/master 2023-09-06T20:33:53,592 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase3.apache.org,45185,1694032430454 2023-09-06T20:33:53,592 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-09-06T20:33:53,592 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44489-0x100eefce21b0001, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-09-06T20:33:53,593 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-09-06T20:33:53,594 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,45185,1694032430454 2023-09-06T20:33:53,597 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-09-06T20:33:53,598 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-09-06T20:33:53,682 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(610): Created cluster ID file at hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/hbase.id with ID: 2bf8fffb-f909-49f8-aba7-44d3113deb23 2023-09-06T20:33:53,723 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:33:53,739 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:33:53,739 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44489-0x100eefce21b0001, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:33:53,783 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:33:53,785 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-09-06T20:33:53,824 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2023-09-06T20:33:53,828 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.RegionGroupingProvider(103): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2023-09-06T20:33:53,840 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(270): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-09-06T20:33:53,840 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(276): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-09-06T20:33:53,841 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(285): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:283) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:310) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.RegionGroupingProvider.init(RegionGroupingProvider.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createProvider(WALFactory.java:186) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:201) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:33:53,852 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.RegionGroupingProvider.init(RegionGroupingProvider.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createProvider(WALFactory.java:186) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:201) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:33:53,924 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/MasterData/data/master/store 2023-09-06T20:33:54,349 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-09-06T20:33:54,349 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:33:54,349 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-09-06T20:33:54,350 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:33:54,350 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:33:54,350 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-09-06T20:33:54,350 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:33:54,350 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:33:54,350 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-09-06T20:33:54,352 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/MasterData/data/master/store/.initializing 2023-09-06T20:33:54,352 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/MasterData/WALs/jenkins-hbase3.apache.org,45185,1694032430454 2023-09-06T20:33:54,354 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-09-06T20:33:54,363 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor jenkins-hbase3.apache.org,45185,1694032430454.regiongroup-0 2023-09-06T20:33:54,377 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C45185%2C1694032430454.jenkins-hbase3.apache.org%2C45185%2C1694032430454.regiongroup-0, suffix=, logDir=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/MasterData/WALs/jenkins-hbase3.apache.org,45185,1694032430454, archiveDir=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/MasterData/oldWALs, maxLogs=10 2023-09-06T20:33:54,411 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/MasterData/WALs/jenkins-hbase3.apache.org,45185,1694032430454/jenkins-hbase3.apache.org%2C45185%2C1694032430454.jenkins-hbase3.apache.org%2C45185%2C1694032430454.regiongroup-0.1694032434384, exclude list is [], retry=0 2023-09-06T20:33:54,440 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK] 2023-09-06T20:33:54,447 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:346) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:430) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:424) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:482) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:477) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:489) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:33:54,517 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/MasterData/WALs/jenkins-hbase3.apache.org,45185,1694032430454/jenkins-hbase3.apache.org%2C45185%2C1694032430454.jenkins-hbase3.apache.org%2C45185%2C1694032430454.regiongroup-0.1694032434384 2023-09-06T20:33:54,517 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK]] 2023-09-06T20:33:54,518 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:33:54,518 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:33:54,523 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:33:54,524 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:33:54,599 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:33:54,634 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-09-06T20:33:54,640 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:33:54,644 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:33:54,644 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:33:54,648 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-09-06T20:33:54,648 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:33:54,649 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:33:54,650 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:33:54,652 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-09-06T20:33:54,653 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:33:54,654 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:33:54,654 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:33:54,656 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-09-06T20:33:54,657 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:33:54,657 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:33:54,663 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:33:54,664 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:33:54,683 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-09-06T20:33:54,686 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:33:54,690 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:33:54,691 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11071593280, jitterRate=0.031122475862503052}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-09-06T20:33:54,697 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-09-06T20:33:54,699 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-09-06T20:33:54,733 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34e1d666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:54,787 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(860): No meta location available on zookeeper, skip migrating... 2023-09-06T20:33:54,807 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-09-06T20:33:54,807 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(563): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-09-06T20:33:54,811 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-09-06T20:33:54,812 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(583): Recovered RegionProcedureStore lease in 1 msec 2023-09-06T20:33:54,818 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(597): Loaded RegionProcedureStore in 6 msec 2023-09-06T20:33:54,819 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-09-06T20:33:54,854 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-09-06T20:33:54,868 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-09-06T20:33:54,871 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-09-06T20:33:54,875 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(150): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-09-06T20:33:54,876 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-09-06T20:33:54,877 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-09-06T20:33:54,880 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-09-06T20:33:54,886 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-09-06T20:33:54,888 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-09-06T20:33:54,889 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-09-06T20:33:54,890 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-09-06T20:33:54,905 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-09-06T20:33:54,908 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-09-06T20:33:54,913 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-09-06T20:33:54,913 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44489-0x100eefce21b0001, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-09-06T20:33:54,913 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:33:54,914 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44489-0x100eefce21b0001, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:33:54,914 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(804): Active/primary master=jenkins-hbase3.apache.org,45185,1694032430454, sessionid=0x100eefce21b0000, setting cluster-up flag (Was=false) 2023-09-06T20:33:54,923 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-09-06T20:33:54,931 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:33:54,931 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44489-0x100eefce21b0001, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:33:54,936 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-09-06T20:33:54,937 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,45185,1694032430454 2023-09-06T20:33:54,942 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:33:54,942 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44489-0x100eefce21b0001, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:33:54,947 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-09-06T20:33:54,948 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,45185,1694032430454 2023-09-06T20:33:54,954 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44489-0x100eefce21b0001, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:33:54,954 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:33:54,962 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-09-06T20:33:54,963 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,45185,1694032430454 2023-09-06T20:33:55,057 DEBUG [RS:0;jenkins-hbase3:44489 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:44489 2023-09-06T20:33:55,059 INFO [RS:0;jenkins-hbase3:44489 {}] regionserver.HRegionServer(997): ClusterId : 2bf8fffb-f909-49f8-aba7-44d3113deb23 2023-09-06T20:33:55,061 INFO [RS:0;jenkins-hbase3:44489 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-09-06T20:33:55,063 DEBUG [RS:0;jenkins-hbase3:44489 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-09-06T20:33:55,069 DEBUG [RS:0;jenkins-hbase3:44489 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-09-06T20:33:55,069 DEBUG [RS:0;jenkins-hbase3:44489 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-09-06T20:33:55,072 DEBUG [RS:0;jenkins-hbase3:44489 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-09-06T20:33:55,072 DEBUG [RS:0;jenkins-hbase3:44489 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-09-06T20:33:55,075 DEBUG [RS:0;jenkins-hbase3:44489 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-09-06T20:33:55,076 DEBUG [RS:0;jenkins-hbase3:44489 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e8ae1f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:55,078 DEBUG [RS:0;jenkins-hbase3:44489 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40a7cfd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-09-06T20:33:55,082 INFO [RS:0;jenkins-hbase3:44489 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-09-06T20:33:55,083 INFO [RS:0;jenkins-hbase3:44489 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-09-06T20:33:55,083 DEBUG [RS:0;jenkins-hbase3:44489 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-09-06T20:33:55,086 INFO [RS:0;jenkins-hbase3:44489 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,45185,1694032430454 with isa=jenkins-hbase3.apache.org/172.31.12.81:44489, startcode=1694032433466 2023-09-06T20:33:55,105 DEBUG [RS:0;jenkins-hbase3:44489 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-09-06T20:33:55,105 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-09-06T20:33:55,114 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-09-06T20:33:55,119 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-09-06T20:33:55,127 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,45185,1694032430454 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-09-06T20:33:55,134 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-06T20:33:55,134 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-06T20:33:55,134 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-06T20:33:55,134 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-06T20:33:55,134 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-09-06T20:33:55,135 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:33:55,135 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-09-06T20:33:55,135 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:33:55,137 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1694032465136 2023-09-06T20:33:55,140 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-09-06T20:33:55,146 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-09-06T20:33:55,149 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-09-06T20:33:55,150 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-09-06T20:33:55,156 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:33:55,157 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-09-06T20:33:55,161 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-09-06T20:33:55,162 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-09-06T20:33:55,162 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-09-06T20:33:55,163 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-09-06T20:33:55,163 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-09-06T20:33:55,176 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:40051, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-09-06T20:33:55,178 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-09-06T20:33:55,181 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-09-06T20:33:55,187 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-09-06T20:33:55,188 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-09-06T20:33:55,192 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-09-06T20:33:55,192 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928 2023-09-06T20:33:55,192 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45185 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3181) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:590) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:33:55,197 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x37c49880 to 127.0.0.1:51523 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:33:55,224 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ab06bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:55,224 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-09-06T20:33:55,228 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:33:55,229 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-09-06T20:33:55,230 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-09-06T20:33:55,231 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-09-06T20:33:55,234 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-09-06T20:33:55,234 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1694032435232,5,FailOnTimeoutGroup] 2023-09-06T20:33:55,234 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:33:55,235 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:33:55,235 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1694032435234,5,FailOnTimeoutGroup] 2023-09-06T20:33:55,235 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-09-06T20:33:55,235 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-09-06T20:33:55,235 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1635): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-09-06T20:33:55,237 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-09-06T20:33:55,237 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-09-06T20:33:55,237 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-09-06T20:33:55,238 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:33:55,238 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:33:55,238 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-09-06T20:33:55,240 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-09-06T20:33:55,240 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:33:55,241 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:33:55,242 DEBUG [RS:0;jenkins-hbase3:44489 {}] regionserver.HRegionServer(3046): Master is not running yet 2023-09-06T20:33:55,242 WARN [RS:0;jenkins-hbase3:44489 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 100 ms and then retrying. 2023-09-06T20:33:55,243 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/hbase/meta/1588230740 2023-09-06T20:33:55,243 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/hbase/meta/1588230740 2023-09-06T20:33:55,246 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-09-06T20:33:55,248 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-09-06T20:33:55,252 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:33:55,253 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10124584480, jitterRate=-0.057074591517448425}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-09-06T20:33:55,256 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-09-06T20:33:55,256 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-09-06T20:33:55,256 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-09-06T20:33:55,256 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-09-06T20:33:55,256 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-09-06T20:33:55,256 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-09-06T20:33:55,258 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-09-06T20:33:55,258 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-09-06T20:33:55,262 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-09-06T20:33:55,262 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-09-06T20:33:55,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-09-06T20:33:55,294 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-09-06T20:33:55,299 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-09-06T20:33:55,343 INFO [RS:0;jenkins-hbase3:44489 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,45185,1694032430454 with isa=jenkins-hbase3.apache.org/172.31.12.81:44489, startcode=1694032433466 2023-09-06T20:33:55,349 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45185 {}] master.ServerManager(403): Registering regionserver=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:33:55,363 DEBUG [RS:0;jenkins-hbase3:44489 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928 2023-09-06T20:33:55,363 DEBUG [RS:0;jenkins-hbase3:44489 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:38135 2023-09-06T20:33:55,363 DEBUG [RS:0;jenkins-hbase3:44489 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=44971 2023-09-06T20:33:55,371 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-09-06T20:33:55,372 DEBUG [RS:0;jenkins-hbase3:44489 {}] zookeeper.ZKUtil(111): regionserver:44489-0x100eefce21b0001, quorum=127.0.0.1:51523, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:33:55,372 WARN [RS:0;jenkins-hbase3:44489 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-09-06T20:33:55,372 INFO [RS:0;jenkins-hbase3:44489 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2023-09-06T20:33:55,372 INFO [RS:0;jenkins-hbase3:44489 {}] wal.RegionGroupingProvider(103): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2023-09-06T20:33:55,372 DEBUG [RS:0;jenkins-hbase3:44489 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:33:55,375 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,44489,1694032433466] 2023-09-06T20:33:55,390 DEBUG [RS:0;jenkins-hbase3:44489 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-09-06T20:33:55,404 INFO [RS:0;jenkins-hbase3:44489 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-09-06T20:33:55,417 INFO [RS:0;jenkins-hbase3:44489 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-09-06T20:33:55,422 INFO [RS:0;jenkins-hbase3:44489 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-09-06T20:33:55,422 INFO [RS:0;jenkins-hbase3:44489 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-09-06T20:33:55,423 INFO [RS:0;jenkins-hbase3:44489 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-09-06T20:33:55,434 INFO [RS:0;jenkins-hbase3:44489 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-09-06T20:33:55,434 DEBUG [RS:0;jenkins-hbase3:44489 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:33:55,434 DEBUG [RS:0;jenkins-hbase3:44489 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:33:55,434 DEBUG [RS:0;jenkins-hbase3:44489 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:33:55,434 DEBUG [RS:0;jenkins-hbase3:44489 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:33:55,434 DEBUG [RS:0;jenkins-hbase3:44489 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:33:55,434 DEBUG [RS:0;jenkins-hbase3:44489 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-09-06T20:33:55,435 DEBUG [RS:0;jenkins-hbase3:44489 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:33:55,435 DEBUG [RS:0;jenkins-hbase3:44489 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:33:55,435 DEBUG [RS:0;jenkins-hbase3:44489 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:33:55,435 DEBUG [RS:0;jenkins-hbase3:44489 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:33:55,435 DEBUG [RS:0;jenkins-hbase3:44489 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:33:55,435 DEBUG [RS:0;jenkins-hbase3:44489 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-09-06T20:33:55,435 DEBUG [RS:0;jenkins-hbase3:44489 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-09-06T20:33:55,436 INFO [RS:0;jenkins-hbase3:44489 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-09-06T20:33:55,436 INFO [RS:0;jenkins-hbase3:44489 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-09-06T20:33:55,436 INFO [RS:0;jenkins-hbase3:44489 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-09-06T20:33:55,437 INFO [RS:0;jenkins-hbase3:44489 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-09-06T20:33:55,437 INFO [RS:0;jenkins-hbase3:44489 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,44489,1694032433466-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-09-06T20:33:55,450 WARN [jenkins-hbase3:45185 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-09-06T20:33:55,452 INFO [RS:0;jenkins-hbase3:44489 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-09-06T20:33:55,455 INFO [RS:0;jenkins-hbase3:44489 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,44489,1694032433466-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-09-06T20:33:55,470 INFO [RS:0;jenkins-hbase3:44489 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,44489,1694032433466 started 2023-09-06T20:33:55,470 INFO [RS:0;jenkins-hbase3:44489 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,44489,1694032433466, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:44489, sessionid=0x100eefce21b0001 2023-09-06T20:33:55,471 DEBUG [RS:0;jenkins-hbase3:44489 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-09-06T20:33:55,471 DEBUG [RS:0;jenkins-hbase3:44489 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:33:55,471 DEBUG [RS:0;jenkins-hbase3:44489 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,44489,1694032433466' 2023-09-06T20:33:55,471 DEBUG [RS:0;jenkins-hbase3:44489 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-09-06T20:33:55,472 DEBUG [RS:0;jenkins-hbase3:44489 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-09-06T20:33:55,472 DEBUG [RS:0;jenkins-hbase3:44489 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-09-06T20:33:55,472 DEBUG [RS:0;jenkins-hbase3:44489 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-09-06T20:33:55,473 DEBUG [RS:0;jenkins-hbase3:44489 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,44489,1694032433466' 2023-09-06T20:33:55,473 DEBUG [RS:0;jenkins-hbase3:44489 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:33:55,473 DEBUG [RS:0;jenkins-hbase3:44489 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:33:55,473 INFO [RS:0;jenkins-hbase3:44489 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-09-06T20:33:55,474 DEBUG [RS:0;jenkins-hbase3:44489 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-09-06T20:33:55,474 DEBUG [RS:0;jenkins-hbase3:44489 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-09-06T20:33:55,474 DEBUG [RS:0;jenkins-hbase3:44489 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:33:55,474 DEBUG [RS:0;jenkins-hbase3:44489 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,44489,1694032433466' 2023-09-06T20:33:55,474 DEBUG [RS:0;jenkins-hbase3:44489 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-09-06T20:33:55,474 DEBUG [RS:0;jenkins-hbase3:44489 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-09-06T20:33:55,475 DEBUG [RS:0;jenkins-hbase3:44489 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-09-06T20:33:55,475 INFO [RS:0;jenkins-hbase3:44489 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-09-06T20:33:55,475 INFO [RS:0;jenkins-hbase3:44489 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-09-06T20:33:55,585 INFO [RS:0;jenkins-hbase3:44489 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-09-06T20:33:55,585 INFO [RS:0;jenkins-hbase3:44489 {}] monitor.StreamSlowMonitor(122): New stream slow monitor jenkins-hbase3.apache.org,44489,1694032433466.regiongroup-0 2023-09-06T20:33:55,588 INFO [RS:0;jenkins-hbase3:44489 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0, suffix=, logDir=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466, archiveDir=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/oldWALs, maxLogs=32 2023-09-06T20:33:55,601 DEBUG [RS:0;jenkins-hbase3:44489 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032435590, exclude list is [], retry=0 2023-09-06T20:33:55,607 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK] 2023-09-06T20:33:55,610 INFO [RS:0;jenkins-hbase3:44489 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032435590 2023-09-06T20:33:55,611 DEBUG [RS:0;jenkins-hbase3:44489 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK]] 2023-09-06T20:33:55,701 DEBUG [jenkins-hbase3:45185 {}] assignment.AssignmentManager(2303): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-09-06T20:33:55,710 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:33:55,716 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,44489,1694032433466, state=OPENING 2023-09-06T20:33:55,724 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-09-06T20:33:55,731 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44489-0x100eefce21b0001, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:33:55,731 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:33:55,731 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-09-06T20:33:55,731 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-09-06T20:33:55,734 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,44489,1694032433466}] 2023-09-06T20:33:55,926 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(721): New admin connection to jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:33:55,929 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-09-06T20:33:55,932 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:35852, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-09-06T20:33:55,943 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-09-06T20:33:55,944 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2023-09-06T20:33:55,944 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.RegionGroupingProvider(103): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2023-09-06T20:33:55,944 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-09-06T20:33:55,945 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-09-06T20:33:55,948 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta, suffix=.meta, logDir=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466, archiveDir=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/oldWALs, maxLogs=32 2023-09-06T20:33:55,960 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta.1694032435949.meta, exclude list is [], retry=0 2023-09-06T20:33:55,965 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK] 2023-09-06T20:33:55,969 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta.1694032435949.meta 2023-09-06T20:33:55,969 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK]] 2023-09-06T20:33:55,969 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:33:55,974 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:33:55,975 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-09-06T20:33:55,992 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-09-06T20:33:55,992 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-09-06T20:33:55,997 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-09-06T20:33:55,998 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:33:55,998 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-09-06T20:33:55,998 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-09-06T20:33:56,000 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-09-06T20:33:56,002 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-09-06T20:33:56,002 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:33:56,002 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:33:56,003 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-09-06T20:33:56,004 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-09-06T20:33:56,004 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:33:56,005 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:33:56,005 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-09-06T20:33:56,006 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-09-06T20:33:56,006 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:33:56,007 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:33:56,009 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/hbase/meta/1588230740 2023-09-06T20:33:56,012 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/hbase/meta/1588230740 2023-09-06T20:33:56,015 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-09-06T20:33:56,017 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-09-06T20:33:56,018 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9810711680, jitterRate=-0.08630627393722534}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-09-06T20:33:56,020 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-09-06T20:33:56,031 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1694032435917 2023-09-06T20:33:56,054 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:33:56,057 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:meta,,1.1588230740 2023-09-06T20:33:56,057 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,44489,1694032433466, state=OPEN 2023-09-06T20:33:56,057 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-09-06T20:33:56,060 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-09-06T20:33:56,060 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44489-0x100eefce21b0001, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-09-06T20:33:56,060 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-09-06T20:33:56,061 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-09-06T20:33:56,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=3, resume processing ppid=2 2023-09-06T20:33:56,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,44489,1694032433466 in 326 msec 2023-09-06T20:33:56,222 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=2, resume processing ppid=1 2023-09-06T20:33:56,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 933 msec 2023-09-06T20:33:56,228 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.1980 sec 2023-09-06T20:33:56,229 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1063): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1694032436229, completionTime=-1 2023-09-06T20:33:56,230 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(830): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-09-06T20:33:56,230 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1629): Joining cluster... 2023-09-06T20:33:56,318 DEBUG [hconnection-0xaacb9cb-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:56,322 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:35856, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:33:56,342 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1641): Number of RegionServers=1 2023-09-06T20:33:56,342 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1694032496342 2023-09-06T20:33:56,342 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1694032556342 2023-09-06T20:33:56,342 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1648): Joined the cluster in 111 msec 2023-09-06T20:33:56,364 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,45185,1694032430454-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-09-06T20:33:56,365 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,45185,1694032430454-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-09-06T20:33:56,365 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,45185,1694032430454-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-09-06T20:33:56,367 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:45185, period=300000, unit=MILLISECONDS is enabled. 2023-09-06T20:33:56,367 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-09-06T20:33:56,373 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-09-06T20:33:56,387 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-09-06T20:33:56,389 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2368): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-09-06T20:33:56,397 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-09-06T20:33:56,401 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:33:56,403 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:33:56,404 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:33:56,823 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => a44b05248e75fb3972b6ec9a34d9f4b8, NAME => 'hbase:namespace,,1694032436388.a44b05248e75fb3972b6ec9a34d9f4b8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928 2023-09-06T20:33:57,235 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1694032436388.a44b05248e75fb3972b6ec9a34d9f4b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:33:57,236 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing a44b05248e75fb3972b6ec9a34d9f4b8, disabling compactions & flushes 2023-09-06T20:33:57,236 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1694032436388.a44b05248e75fb3972b6ec9a34d9f4b8. 2023-09-06T20:33:57,236 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1694032436388.a44b05248e75fb3972b6ec9a34d9f4b8. 2023-09-06T20:33:57,236 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1694032436388.a44b05248e75fb3972b6ec9a34d9f4b8. after waiting 0 ms 2023-09-06T20:33:57,236 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1694032436388.a44b05248e75fb3972b6ec9a34d9f4b8. 2023-09-06T20:33:57,236 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1694032436388.a44b05248e75fb3972b6ec9a34d9f4b8. 2023-09-06T20:33:57,236 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for a44b05248e75fb3972b6ec9a34d9f4b8: 2023-09-06T20:33:57,238 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:33:57,252 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1694032436388.a44b05248e75fb3972b6ec9a34d9f4b8.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1694032437240"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032437240"}]},"ts":"1694032437240"} 2023-09-06T20:33:57,286 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:33:57,287 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:33:57,293 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032437288"}]},"ts":"1694032437288"} 2023-09-06T20:33:57,297 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-09-06T20:33:57,303 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=a44b05248e75fb3972b6ec9a34d9f4b8, ASSIGN}] 2023-09-06T20:33:57,305 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=a44b05248e75fb3972b6ec9a34d9f4b8, ASSIGN 2023-09-06T20:33:57,307 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=a44b05248e75fb3972b6ec9a34d9f4b8, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44489,1694032433466; forceNewPlan=false, retain=false 2023-09-06T20:33:57,458 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=a44b05248e75fb3972b6ec9a34d9f4b8, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:33:57,465 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure a44b05248e75fb3972b6ec9a34d9f4b8, server=jenkins-hbase3.apache.org,44489,1694032433466}] 2023-09-06T20:33:57,624 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1694032436388.a44b05248e75fb3972b6ec9a34d9f4b8. 2023-09-06T20:33:57,624 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-09-06T20:33:57,625 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] monitor.StreamSlowMonitor(122): New stream slow monitor jenkins-hbase3.apache.org,44489,1694032433466.regiongroup-1 2023-09-06T20:33:57,628 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1, suffix=, logDir=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466, archiveDir=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/oldWALs, maxLogs=32 2023-09-06T20:33:57,640 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032437629, exclude list is [], retry=0 2023-09-06T20:33:57,644 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK] 2023-09-06T20:33:57,648 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032437629 2023-09-06T20:33:57,648 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK]] 2023-09-06T20:33:57,649 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => a44b05248e75fb3972b6ec9a34d9f4b8, NAME => 'hbase:namespace,,1694032436388.a44b05248e75fb3972b6ec9a34d9f4b8.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:33:57,650 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:33:57,650 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace a44b05248e75fb3972b6ec9a34d9f4b8 2023-09-06T20:33:57,650 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1694032436388.a44b05248e75fb3972b6ec9a34d9f4b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:33:57,650 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for a44b05248e75fb3972b6ec9a34d9f4b8 2023-09-06T20:33:57,650 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for a44b05248e75fb3972b6ec9a34d9f4b8 2023-09-06T20:33:57,652 INFO [StoreOpener-a44b05248e75fb3972b6ec9a34d9f4b8-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region a44b05248e75fb3972b6ec9a34d9f4b8 2023-09-06T20:33:57,655 INFO [StoreOpener-a44b05248e75fb3972b6ec9a34d9f4b8-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a44b05248e75fb3972b6ec9a34d9f4b8 columnFamilyName info 2023-09-06T20:33:57,655 DEBUG [StoreOpener-a44b05248e75fb3972b6ec9a34d9f4b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:33:57,655 INFO [StoreOpener-a44b05248e75fb3972b6ec9a34d9f4b8-1 {}] regionserver.HStore(324): Store=a44b05248e75fb3972b6ec9a34d9f4b8/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:33:57,657 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/hbase/namespace/a44b05248e75fb3972b6ec9a34d9f4b8 2023-09-06T20:33:57,658 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/hbase/namespace/a44b05248e75fb3972b6ec9a34d9f4b8 2023-09-06T20:33:57,661 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for a44b05248e75fb3972b6ec9a34d9f4b8 2023-09-06T20:33:57,664 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/hbase/namespace/a44b05248e75fb3972b6ec9a34d9f4b8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:33:57,665 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened a44b05248e75fb3972b6ec9a34d9f4b8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9610138240, jitterRate=-0.10498613119125366}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:33:57,666 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for a44b05248e75fb3972b6ec9a34d9f4b8: 2023-09-06T20:33:57,668 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:namespace,,1694032436388.a44b05248e75fb3972b6ec9a34d9f4b8., pid=6, masterSystemTime=1694032437618 2023-09-06T20:33:57,672 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:namespace,,1694032436388.a44b05248e75fb3972b6ec9a34d9f4b8. 2023-09-06T20:33:57,672 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1694032436388.a44b05248e75fb3972b6ec9a34d9f4b8. 2023-09-06T20:33:57,673 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=a44b05248e75fb3972b6ec9a34d9f4b8, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:33:57,679 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=6, resume processing ppid=5 2023-09-06T20:33:57,679 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure a44b05248e75fb3972b6ec9a34d9f4b8, server=jenkins-hbase3.apache.org,44489,1694032433466 in 212 msec 2023-09-06T20:33:57,682 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=5, resume processing ppid=4 2023-09-06T20:33:57,682 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=a44b05248e75fb3972b6ec9a34d9f4b8, ASSIGN in 376 msec 2023-09-06T20:33:57,683 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:33:57,684 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032437683"}]},"ts":"1694032437683"} 2023-09-06T20:33:57,687 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-09-06T20:33:57,690 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:33:57,692 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 1.3000 sec 2023-09-06T20:33:57,705 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-09-06T20:33:57,707 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44489-0x100eefce21b0001, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:33:57,707 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-09-06T20:33:57,707 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:33:57,752 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-09-06T20:33:57,768 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:33:57,774 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 28 msec 2023-09-06T20:33:57,777 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-09-06T20:33:57,791 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:33:57,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 18 msec 2023-09-06T20:33:57,803 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-09-06T20:33:57,806 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-09-06T20:33:57,807 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1193): Master has completed initialization 4.212sec 2023-09-06T20:33:57,809 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-09-06T20:33:57,811 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-09-06T20:33:57,813 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-09-06T20:33:57,813 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-09-06T20:33:57,814 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-09-06T20:33:57,816 DEBUG [Listener at localhost/46585 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x448dd8c9 to 127.0.0.1:51523 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:33:57,816 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,45185,1694032430454-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-09-06T20:33:57,817 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,45185,1694032430454-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-09-06T20:33:57,820 DEBUG [Listener at localhost/46585 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a336b30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:33:57,827 DEBUG [Listener at localhost/46585 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-09-06T20:33:57,827 DEBUG [Listener at localhost/46585 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-09-06T20:33:57,834 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1296): Balancer post startup initialization complete, took 0 seconds 2023-09-06T20:33:57,836 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-09-06T20:33:57,845 DEBUG [hconnection-0x39685d06-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:33:57,856 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:35862, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:33:57,868 INFO [Listener at localhost/46585 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,45185,1694032430454 2023-09-06T20:33:57,868 INFO [Listener at localhost/46585 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-09-06T20:33:57,868 INFO [Listener at localhost/46585 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/test.cache.data in system properties and HBase conf 2023-09-06T20:33:57,868 INFO [Listener at localhost/46585 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/hadoop.tmp.dir in system properties and HBase conf 2023-09-06T20:33:57,868 INFO [Listener at localhost/46585 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/hadoop.log.dir in system properties and HBase conf 2023-09-06T20:33:57,868 INFO [Listener at localhost/46585 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/mapreduce.cluster.local.dir in system properties and HBase conf 2023-09-06T20:33:57,868 INFO [Listener at localhost/46585 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-09-06T20:33:57,869 INFO [Listener at localhost/46585 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-09-06T20:33:57,869 INFO [Listener at localhost/46585 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-09-06T20:33:57,869 INFO [Listener at localhost/46585 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-09-06T20:33:57,869 INFO [Listener at localhost/46585 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-09-06T20:33:57,869 INFO [Listener at localhost/46585 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-06T20:33:57,869 INFO [Listener at localhost/46585 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-09-06T20:33:57,869 INFO [Listener at localhost/46585 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-09-06T20:33:57,869 INFO [Listener at localhost/46585 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-06T20:33:57,869 INFO [Listener at localhost/46585 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-06T20:33:57,869 INFO [Listener at localhost/46585 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-09-06T20:33:57,869 INFO [Listener at localhost/46585 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/nfs.dump.dir in system properties and HBase conf 2023-09-06T20:33:57,869 INFO [Listener at localhost/46585 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/java.io.tmpdir in system properties and HBase conf 2023-09-06T20:33:57,869 INFO [Listener at localhost/46585 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-06T20:33:57,870 INFO [Listener at localhost/46585 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-09-06T20:33:57,870 INFO [Listener at localhost/46585 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-09-06T20:33:59,347 WARN [Thread-251 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:33:59,712 INFO [Thread-251 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:33:59,717 INFO [Thread-251 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_39335_jobhistory____.ndj03r/webapp 2023-09-06T20:33:59,883 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-09-06T20:33:59,884 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-09-06T20:33:59,884 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-09-06T20:33:59,887 INFO [Thread-251 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-09-06T20:33:59,982 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:34:00,099 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:34:00,351 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-09-06T20:34:00,380 INFO [Thread-251 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:39335 2023-09-06T20:34:00,844 WARN [Listener at jenkins-hbase3.apache.org/32975 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:34:00,928 WARN [Listener at jenkins-hbase3.apache.org/32975 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-09-06T20:34:00,929 INFO [Listener at jenkins-hbase3.apache.org/32975 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:34:00,938 INFO [Listener at jenkins-hbase3.apache.org/32975 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_44333_cluster____lvgc8g/webapp 2023-09-06T20:34:01,037 INFO [Listener at jenkins-hbase3.apache.org/32975 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-09-06T20:34:01,037 INFO [Listener at jenkins-hbase3.apache.org/32975 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-09-06T20:34:01,037 INFO [Listener at jenkins-hbase3.apache.org/32975 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-09-06T20:34:01,037 INFO [Listener at jenkins-hbase3.apache.org/32975 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-09-06T20:34:01,047 INFO [Listener at jenkins-hbase3.apache.org/32975 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:34:01,245 INFO [Listener at jenkins-hbase3.apache.org/32975 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:34:01,325 INFO [Listener at jenkins-hbase3.apache.org/32975 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:34:01,332 INFO [Listener at jenkins-hbase3.apache.org/32975 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:44333 2023-09-06T20:34:01,534 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:34:01,612 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-09-06T20:34:01,614 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-09-06T20:34:01,615 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-09-06T20:34:01,615 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-09-06T20:34:01,617 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-09-06T20:34:01,617 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-09-06T20:34:01,618 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:34:01,618 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-09-06T20:34:01,618 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-09-06T20:34:01,618 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-09-06T20:34:01,618 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-09-06T20:34:01,619 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-09-06T20:34:01,765 WARN [Listener at jenkins-hbase3.apache.org/34511 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:34:01,788 WARN [Listener at jenkins-hbase3.apache.org/34511 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-09-06T20:34:01,788 INFO [Listener at jenkins-hbase3.apache.org/34511 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:34:01,795 INFO [Listener at jenkins-hbase3.apache.org/34511 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_44947_node____.8nevvp/webapp 2023-09-06T20:34:01,848 INFO [Listener at jenkins-hbase3.apache.org/34511 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-09-06T20:34:01,848 INFO [Listener at jenkins-hbase3.apache.org/34511 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-09-06T20:34:01,848 INFO [Listener at jenkins-hbase3.apache.org/34511 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-09-06T20:34:01,848 INFO [Listener at jenkins-hbase3.apache.org/34511 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-09-06T20:34:01,856 INFO [Listener at jenkins-hbase3.apache.org/34511 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:34:01,877 INFO [Listener at jenkins-hbase3.apache.org/34511 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:34:01,948 INFO [Listener at jenkins-hbase3.apache.org/34511 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:34:01,958 INFO [Listener at jenkins-hbase3.apache.org/34511 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:44947 2023-09-06T20:34:02,105 WARN [Listener at jenkins-hbase3.apache.org/45863 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:34:02,111 WARN [Listener at jenkins-hbase3.apache.org/45863 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-09-06T20:34:02,111 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:34:02,117 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_37191_node____.mqayt8/webapp 2023-09-06T20:34:02,166 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-09-06T20:34:02,166 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-09-06T20:34:02,167 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-09-06T20:34:02,167 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-09-06T20:34:02,174 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:34:02,189 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:34:02,252 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:34:02,258 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:37191 2023-09-06T20:34:02,288 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-09-06T20:34:02,288 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:38135/backupUT 2023-09-06T20:34:02,300 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:34:02,304 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:47152, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:34:02,314 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns1'} 2023-09-06T20:34:02,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-09-06T20:34:02,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-09-06T20:34:02,341 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:34:02,346 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 29 msec 2023-09-06T20:34:02,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-09-06T20:34:02,437 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns2'} 2023-09-06T20:34:02,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-09-06T20:34:02,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=10 2023-09-06T20:34:02,451 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:34:02,459 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 20 msec 2023-09-06T20:34:02,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=10 2023-09-06T20:34:02,547 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns3'} 2023-09-06T20:34:02,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-09-06T20:34:02,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=11 2023-09-06T20:34:02,560 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:34:02,565 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 16 msec 2023-09-06T20:34:02,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=11 2023-09-06T20:34:02,657 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns4'} 2023-09-06T20:34:02,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-09-06T20:34:02,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-09-06T20:34:02,669 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:34:02,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 17 msec 2023-09-06T20:34:02,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-09-06T20:34:02,778 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'test-1694032442288', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:34:02,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1694032442288 2023-09-06T20:34:02,782 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1694032442288 execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:34:02,783 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "test-1694032442288" procId is: 13 2023-09-06T20:34:02,783 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:34:02,784 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1694032442288 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:34:02,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-09-06T20:34:02,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-09-06T20:34:03,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-09-06T20:34:03,223 INFO [RegionOpenAndInit-test-1694032442288-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 90d54d530e57af59206a5a1415c17690, NAME => 'test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1694032442288', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928 2023-09-06T20:34:03,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-09-06T20:34:03,642 DEBUG [RegionOpenAndInit-test-1694032442288-pool-0 {}] regionserver.HRegion(888): Instantiated test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:34:03,642 DEBUG [RegionOpenAndInit-test-1694032442288-pool-0 {}] regionserver.HRegion(1668): Closing 90d54d530e57af59206a5a1415c17690, disabling compactions & flushes 2023-09-06T20:34:03,642 INFO [RegionOpenAndInit-test-1694032442288-pool-0 {}] regionserver.HRegion(1690): Closing region test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690. 2023-09-06T20:34:03,642 DEBUG [RegionOpenAndInit-test-1694032442288-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690. 2023-09-06T20:34:03,642 DEBUG [RegionOpenAndInit-test-1694032442288-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690. after waiting 0 ms 2023-09-06T20:34:03,642 DEBUG [RegionOpenAndInit-test-1694032442288-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690. 2023-09-06T20:34:03,642 INFO [RegionOpenAndInit-test-1694032442288-pool-0 {}] regionserver.HRegion(1909): Closed test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690. 2023-09-06T20:34:03,642 DEBUG [RegionOpenAndInit-test-1694032442288-pool-0 {}] regionserver.HRegion(1622): Region close journal for 90d54d530e57af59206a5a1415c17690: 2023-09-06T20:34:03,644 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1694032442288 execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:34:03,644 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1694032443644"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032443644"}]},"ts":"1694032443644"} 2023-09-06T20:34:03,647 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:34:03,648 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1694032442288 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:34:03,648 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1694032442288","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032443648"}]},"ts":"1694032443648"} 2023-09-06T20:34:03,650 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1694032442288, state=ENABLING in hbase:meta 2023-09-06T20:34:03,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1694032442288, region=90d54d530e57af59206a5a1415c17690, ASSIGN}] 2023-09-06T20:34:03,658 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1694032442288, region=90d54d530e57af59206a5a1415c17690, ASSIGN 2023-09-06T20:34:03,659 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1694032442288, region=90d54d530e57af59206a5a1415c17690, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44489,1694032433466; forceNewPlan=false, retain=false 2023-09-06T20:34:03,810 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=90d54d530e57af59206a5a1415c17690, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:03,814 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure 90d54d530e57af59206a5a1415c17690, server=jenkins-hbase3.apache.org,44489,1694032433466}] 2023-09-06T20:34:03,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-09-06T20:34:03,974 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690. 2023-09-06T20:34:03,975 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => 90d54d530e57af59206a5a1415c17690, NAME => 'test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:34:03,975 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:34:03,975 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1694032442288 90d54d530e57af59206a5a1415c17690 2023-09-06T20:34:03,975 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:34:03,975 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for 90d54d530e57af59206a5a1415c17690 2023-09-06T20:34:03,975 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for 90d54d530e57af59206a5a1415c17690 2023-09-06T20:34:03,978 INFO [StoreOpener-90d54d530e57af59206a5a1415c17690-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 90d54d530e57af59206a5a1415c17690 2023-09-06T20:34:03,980 INFO [StoreOpener-90d54d530e57af59206a5a1415c17690-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90d54d530e57af59206a5a1415c17690 columnFamilyName f 2023-09-06T20:34:03,980 DEBUG [StoreOpener-90d54d530e57af59206a5a1415c17690-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:34:03,981 INFO [StoreOpener-90d54d530e57af59206a5a1415c17690-1 {}] regionserver.HStore(324): Store=90d54d530e57af59206a5a1415c17690/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:34:03,982 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/90d54d530e57af59206a5a1415c17690 2023-09-06T20:34:03,983 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/90d54d530e57af59206a5a1415c17690 2023-09-06T20:34:03,989 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for 90d54d530e57af59206a5a1415c17690 2023-09-06T20:34:03,992 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/90d54d530e57af59206a5a1415c17690/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:34:03,993 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened 90d54d530e57af59206a5a1415c17690; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11996742400, jitterRate=0.11728370189666748}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:34:03,994 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for 90d54d530e57af59206a5a1415c17690: 2023-09-06T20:34:03,995 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690., pid=15, masterSystemTime=1694032443967 2023-09-06T20:34:03,998 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690. 2023-09-06T20:34:03,998 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690. 2023-09-06T20:34:03,999 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=90d54d530e57af59206a5a1415c17690, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:04,004 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=15, resume processing ppid=14 2023-09-06T20:34:04,004 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure 90d54d530e57af59206a5a1415c17690, server=jenkins-hbase3.apache.org,44489,1694032433466 in 187 msec 2023-09-06T20:34:04,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=14, resume processing ppid=13 2023-09-06T20:34:04,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1694032442288, region=90d54d530e57af59206a5a1415c17690, ASSIGN in 349 msec 2023-09-06T20:34:04,008 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1694032442288 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:34:04,008 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1694032442288","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032444008"}]},"ts":"1694032444008"} 2023-09-06T20:34:04,010 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1694032442288, state=ENABLED in hbase:meta 2023-09-06T20:34:04,013 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1694032442288 execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:34:04,015 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1694032442288 in 1.2350 sec 2023-09-06T20:34:04,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-09-06T20:34:04,902 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:test-1694032442288, procId: 13 completed 2023-09-06T20:34:04,903 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1f9ce6ad to 127.0.0.1:51523 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:34:04,910 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78cbfcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:04,912 DEBUG [hconnection-0x15975daf-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:04,915 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:35410, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:04,920 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44489 {}] regionserver.HRegion(8210): writing data to region test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690. with WAL disabled. Data may be lost in the event of a crash. 2023-09-06T20:34:05,019 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns2:test-16940324422881', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:34:05,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-16940324422881 2023-09-06T20:34:05,023 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16940324422881 execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:34:05,023 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:34:05,023 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "test-16940324422881" procId is: 16 2023-09-06T20:34:05,024 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-16940324422881 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:34:05,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:34:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:34:05,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:34:05,440 INFO [RegionOpenAndInit-ns2:test-16940324422881-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 95242ef863d74dd33bf9cf6336515789, NAME => 'ns2:test-16940324422881,,1694032445019.95242ef863d74dd33bf9cf6336515789.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16940324422881', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928 2023-09-06T20:34:05,454 DEBUG [RegionOpenAndInit-ns2:test-16940324422881-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16940324422881,,1694032445019.95242ef863d74dd33bf9cf6336515789.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:34:05,454 DEBUG [RegionOpenAndInit-ns2:test-16940324422881-pool-0 {}] regionserver.HRegion(1668): Closing 95242ef863d74dd33bf9cf6336515789, disabling compactions & flushes 2023-09-06T20:34:05,454 INFO [RegionOpenAndInit-ns2:test-16940324422881-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16940324422881,,1694032445019.95242ef863d74dd33bf9cf6336515789. 2023-09-06T20:34:05,455 DEBUG [RegionOpenAndInit-ns2:test-16940324422881-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16940324422881,,1694032445019.95242ef863d74dd33bf9cf6336515789. 2023-09-06T20:34:05,455 DEBUG [RegionOpenAndInit-ns2:test-16940324422881-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16940324422881,,1694032445019.95242ef863d74dd33bf9cf6336515789. after waiting 0 ms 2023-09-06T20:34:05,455 DEBUG [RegionOpenAndInit-ns2:test-16940324422881-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16940324422881,,1694032445019.95242ef863d74dd33bf9cf6336515789. 2023-09-06T20:34:05,455 INFO [RegionOpenAndInit-ns2:test-16940324422881-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16940324422881,,1694032445019.95242ef863d74dd33bf9cf6336515789. 2023-09-06T20:34:05,455 DEBUG [RegionOpenAndInit-ns2:test-16940324422881-pool-0 {}] regionserver.HRegion(1622): Region close journal for 95242ef863d74dd33bf9cf6336515789: 2023-09-06T20:34:05,456 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-16940324422881 execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:34:05,457 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16940324422881,,1694032445019.95242ef863d74dd33bf9cf6336515789.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1694032445456"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032445456"}]},"ts":"1694032445456"} 2023-09-06T20:34:05,459 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:34:05,460 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-16940324422881 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:34:05,461 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16940324422881","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032445460"}]},"ts":"1694032445460"} 2023-09-06T20:34:05,462 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16940324422881, state=ENABLING in hbase:meta 2023-09-06T20:34:05,467 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16940324422881, region=95242ef863d74dd33bf9cf6336515789, ASSIGN}] 2023-09-06T20:34:05,468 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16940324422881, region=95242ef863d74dd33bf9cf6336515789, ASSIGN 2023-09-06T20:34:05,469 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16940324422881, region=95242ef863d74dd33bf9cf6336515789, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44489,1694032433466; forceNewPlan=false, retain=false 2023-09-06T20:34:05,621 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=95242ef863d74dd33bf9cf6336515789, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:05,624 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure 95242ef863d74dd33bf9cf6336515789, server=jenkins-hbase3.apache.org,44489,1694032433466}] 2023-09-06T20:34:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:34:05,781 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-16940324422881,,1694032445019.95242ef863d74dd33bf9cf6336515789. 2023-09-06T20:34:05,781 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => 95242ef863d74dd33bf9cf6336515789, NAME => 'ns2:test-16940324422881,,1694032445019.95242ef863d74dd33bf9cf6336515789.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:34:05,782 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:34:05,782 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16940324422881 95242ef863d74dd33bf9cf6336515789 2023-09-06T20:34:05,782 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-16940324422881,,1694032445019.95242ef863d74dd33bf9cf6336515789.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:34:05,782 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for 95242ef863d74dd33bf9cf6336515789 2023-09-06T20:34:05,782 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for 95242ef863d74dd33bf9cf6336515789 2023-09-06T20:34:05,784 INFO [StoreOpener-95242ef863d74dd33bf9cf6336515789-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 95242ef863d74dd33bf9cf6336515789 2023-09-06T20:34:05,786 INFO [StoreOpener-95242ef863d74dd33bf9cf6336515789-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 95242ef863d74dd33bf9cf6336515789 columnFamilyName f 2023-09-06T20:34:05,786 DEBUG [StoreOpener-95242ef863d74dd33bf9cf6336515789-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:34:05,787 INFO [StoreOpener-95242ef863d74dd33bf9cf6336515789-1 {}] regionserver.HStore(324): Store=95242ef863d74dd33bf9cf6336515789/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:34:05,788 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/ns2/test-16940324422881/95242ef863d74dd33bf9cf6336515789 2023-09-06T20:34:05,789 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/ns2/test-16940324422881/95242ef863d74dd33bf9cf6336515789 2023-09-06T20:34:05,792 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for 95242ef863d74dd33bf9cf6336515789 2023-09-06T20:34:05,799 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/ns2/test-16940324422881/95242ef863d74dd33bf9cf6336515789/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:34:05,800 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened 95242ef863d74dd33bf9cf6336515789; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10412306720, jitterRate=-0.030278369784355164}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:34:05,801 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for 95242ef863d74dd33bf9cf6336515789: 2023-09-06T20:34:05,802 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:test-16940324422881,,1694032445019.95242ef863d74dd33bf9cf6336515789., pid=18, masterSystemTime=1694032445776 2023-09-06T20:34:05,805 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:test-16940324422881,,1694032445019.95242ef863d74dd33bf9cf6336515789. 2023-09-06T20:34:05,805 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-16940324422881,,1694032445019.95242ef863d74dd33bf9cf6336515789. 2023-09-06T20:34:05,806 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=95242ef863d74dd33bf9cf6336515789, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:05,810 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=18, resume processing ppid=17 2023-09-06T20:34:05,810 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure 95242ef863d74dd33bf9cf6336515789, server=jenkins-hbase3.apache.org,44489,1694032433466 in 184 msec 2023-09-06T20:34:05,813 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=17, resume processing ppid=16 2023-09-06T20:34:05,813 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16940324422881, region=95242ef863d74dd33bf9cf6336515789, ASSIGN in 344 msec 2023-09-06T20:34:05,814 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-16940324422881 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:34:05,814 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16940324422881","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032445814"}]},"ts":"1694032445814"} 2023-09-06T20:34:05,816 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16940324422881, state=ENABLED in hbase:meta 2023-09-06T20:34:05,818 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16940324422881 execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:34:05,820 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-16940324422881 in 800 msec 2023-09-06T20:34:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:34:06,131 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns2:test-16940324422881, procId: 16 completed 2023-09-06T20:34:06,136 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44489 {}] regionserver.HRegion(8210): writing data to region ns2:test-16940324422881,,1694032445019.95242ef863d74dd33bf9cf6336515789. with WAL disabled. Data may be lost in the event of a crash. 2023-09-06T20:34:06,224 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns3:test-16940324422882', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:34:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-16940324422882 2023-09-06T20:34:06,228 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16940324422882 execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:34:06,228 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:34:06,228 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "test-16940324422882" procId is: 19 2023-09-06T20:34:06,229 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-16940324422882 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:34:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-09-06T20:34:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-09-06T20:34:06,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-09-06T20:34:06,645 INFO [RegionOpenAndInit-ns3:test-16940324422882-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => d8fb0a006da661f8e8078937dc326385, NAME => 'ns3:test-16940324422882,,1694032446224.d8fb0a006da661f8e8078937dc326385.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16940324422882', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928 2023-09-06T20:34:06,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-09-06T20:34:07,060 DEBUG [RegionOpenAndInit-ns3:test-16940324422882-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16940324422882,,1694032446224.d8fb0a006da661f8e8078937dc326385.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:34:07,060 DEBUG [RegionOpenAndInit-ns3:test-16940324422882-pool-0 {}] regionserver.HRegion(1668): Closing d8fb0a006da661f8e8078937dc326385, disabling compactions & flushes 2023-09-06T20:34:07,060 INFO [RegionOpenAndInit-ns3:test-16940324422882-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16940324422882,,1694032446224.d8fb0a006da661f8e8078937dc326385. 2023-09-06T20:34:07,060 DEBUG [RegionOpenAndInit-ns3:test-16940324422882-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16940324422882,,1694032446224.d8fb0a006da661f8e8078937dc326385. 2023-09-06T20:34:07,060 DEBUG [RegionOpenAndInit-ns3:test-16940324422882-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16940324422882,,1694032446224.d8fb0a006da661f8e8078937dc326385. after waiting 0 ms 2023-09-06T20:34:07,060 DEBUG [RegionOpenAndInit-ns3:test-16940324422882-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16940324422882,,1694032446224.d8fb0a006da661f8e8078937dc326385. 2023-09-06T20:34:07,060 INFO [RegionOpenAndInit-ns3:test-16940324422882-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16940324422882,,1694032446224.d8fb0a006da661f8e8078937dc326385. 2023-09-06T20:34:07,060 DEBUG [RegionOpenAndInit-ns3:test-16940324422882-pool-0 {}] regionserver.HRegion(1622): Region close journal for d8fb0a006da661f8e8078937dc326385: 2023-09-06T20:34:07,062 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-16940324422882 execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:34:07,062 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16940324422882,,1694032446224.d8fb0a006da661f8e8078937dc326385.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1694032447062"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032447062"}]},"ts":"1694032447062"} 2023-09-06T20:34:07,064 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:34:07,066 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-16940324422882 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:34:07,066 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16940324422882","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032447066"}]},"ts":"1694032447066"} 2023-09-06T20:34:07,067 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16940324422882, state=ENABLING in hbase:meta 2023-09-06T20:34:07,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16940324422882, region=d8fb0a006da661f8e8078937dc326385, ASSIGN}] 2023-09-06T20:34:07,075 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16940324422882, region=d8fb0a006da661f8e8078937dc326385, ASSIGN 2023-09-06T20:34:07,076 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16940324422882, region=d8fb0a006da661f8e8078937dc326385, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44489,1694032433466; forceNewPlan=false, retain=false 2023-09-06T20:34:07,227 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=d8fb0a006da661f8e8078937dc326385, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:07,230 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure d8fb0a006da661f8e8078937dc326385, server=jenkins-hbase3.apache.org,44489,1694032433466}] 2023-09-06T20:34:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-09-06T20:34:07,387 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-16940324422882,,1694032446224.d8fb0a006da661f8e8078937dc326385. 2023-09-06T20:34:07,387 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => d8fb0a006da661f8e8078937dc326385, NAME => 'ns3:test-16940324422882,,1694032446224.d8fb0a006da661f8e8078937dc326385.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:34:07,387 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:34:07,387 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16940324422882 d8fb0a006da661f8e8078937dc326385 2023-09-06T20:34:07,388 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-16940324422882,,1694032446224.d8fb0a006da661f8e8078937dc326385.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:34:07,388 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for d8fb0a006da661f8e8078937dc326385 2023-09-06T20:34:07,388 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for d8fb0a006da661f8e8078937dc326385 2023-09-06T20:34:07,389 INFO [StoreOpener-d8fb0a006da661f8e8078937dc326385-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region d8fb0a006da661f8e8078937dc326385 2023-09-06T20:34:07,391 INFO [StoreOpener-d8fb0a006da661f8e8078937dc326385-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d8fb0a006da661f8e8078937dc326385 columnFamilyName f 2023-09-06T20:34:07,392 DEBUG [StoreOpener-d8fb0a006da661f8e8078937dc326385-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:34:07,392 INFO [StoreOpener-d8fb0a006da661f8e8078937dc326385-1 {}] regionserver.HStore(324): Store=d8fb0a006da661f8e8078937dc326385/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:34:07,393 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/ns3/test-16940324422882/d8fb0a006da661f8e8078937dc326385 2023-09-06T20:34:07,394 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/ns3/test-16940324422882/d8fb0a006da661f8e8078937dc326385 2023-09-06T20:34:07,398 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for d8fb0a006da661f8e8078937dc326385 2023-09-06T20:34:07,401 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/ns3/test-16940324422882/d8fb0a006da661f8e8078937dc326385/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:34:07,402 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened d8fb0a006da661f8e8078937dc326385; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11913101760, jitterRate=0.10949406027793884}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:34:07,403 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for d8fb0a006da661f8e8078937dc326385: 2023-09-06T20:34:07,404 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:test-16940324422882,,1694032446224.d8fb0a006da661f8e8078937dc326385., pid=21, masterSystemTime=1694032447382 2023-09-06T20:34:07,407 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:test-16940324422882,,1694032446224.d8fb0a006da661f8e8078937dc326385. 2023-09-06T20:34:07,407 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-16940324422882,,1694032446224.d8fb0a006da661f8e8078937dc326385. 2023-09-06T20:34:07,408 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=d8fb0a006da661f8e8078937dc326385, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:07,413 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=21, resume processing ppid=20 2023-09-06T20:34:07,413 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure d8fb0a006da661f8e8078937dc326385, server=jenkins-hbase3.apache.org,44489,1694032433466 in 181 msec 2023-09-06T20:34:07,415 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=20, resume processing ppid=19 2023-09-06T20:34:07,415 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16940324422882, region=d8fb0a006da661f8e8078937dc326385, ASSIGN in 340 msec 2023-09-06T20:34:07,416 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-16940324422882 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:34:07,416 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16940324422882","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032447416"}]},"ts":"1694032447416"} 2023-09-06T20:34:07,418 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16940324422882, state=ENABLED in hbase:meta 2023-09-06T20:34:07,422 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16940324422882 execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:34:07,424 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-16940324422882 in 1.1980 sec 2023-09-06T20:34:07,615 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16940324422882' 2023-09-06T20:34:07,616 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-16940324422881' 2023-09-06T20:34:07,617 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1694032442288' 2023-09-06T20:34:08,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-09-06T20:34:08,338 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns3:test-16940324422882, procId: 19 completed 2023-09-06T20:34:08,338 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-16940324422882 get assigned. Timeout = 60000ms 2023-09-06T20:34:08,340 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-06T20:34:08,345 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-16940324422882 assigned to meta. Checking AM states. 2023-09-06T20:34:08,345 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-06T20:34:08,346 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-16940324422882 assigned. 2023-09-06T20:34:08,347 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns4:test-16940324422883', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:34:08,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-16940324422883 2023-09-06T20:34:08,351 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16940324422883 execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:34:08,351 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:34:08,351 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns4" qualifier: "test-16940324422883" procId is: 22 2023-09-06T20:34:08,352 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-16940324422883 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:34:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:34:08,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:34:08,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:34:08,767 INFO [RegionOpenAndInit-ns4:test-16940324422883-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 2fb717bf2da453848b7afb361b8ed8c0, NAME => 'ns4:test-16940324422883,,1694032448347.2fb717bf2da453848b7afb361b8ed8c0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-16940324422883', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928 2023-09-06T20:34:08,783 DEBUG [RegionOpenAndInit-ns4:test-16940324422883-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-16940324422883,,1694032448347.2fb717bf2da453848b7afb361b8ed8c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:34:08,784 DEBUG [RegionOpenAndInit-ns4:test-16940324422883-pool-0 {}] regionserver.HRegion(1668): Closing 2fb717bf2da453848b7afb361b8ed8c0, disabling compactions & flushes 2023-09-06T20:34:08,784 INFO [RegionOpenAndInit-ns4:test-16940324422883-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-16940324422883,,1694032448347.2fb717bf2da453848b7afb361b8ed8c0. 2023-09-06T20:34:08,784 DEBUG [RegionOpenAndInit-ns4:test-16940324422883-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16940324422883,,1694032448347.2fb717bf2da453848b7afb361b8ed8c0. 2023-09-06T20:34:08,784 DEBUG [RegionOpenAndInit-ns4:test-16940324422883-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16940324422883,,1694032448347.2fb717bf2da453848b7afb361b8ed8c0. after waiting 0 ms 2023-09-06T20:34:08,784 DEBUG [RegionOpenAndInit-ns4:test-16940324422883-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16940324422883,,1694032448347.2fb717bf2da453848b7afb361b8ed8c0. 2023-09-06T20:34:08,784 INFO [RegionOpenAndInit-ns4:test-16940324422883-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-16940324422883,,1694032448347.2fb717bf2da453848b7afb361b8ed8c0. 2023-09-06T20:34:08,784 DEBUG [RegionOpenAndInit-ns4:test-16940324422883-pool-0 {}] regionserver.HRegion(1622): Region close journal for 2fb717bf2da453848b7afb361b8ed8c0: 2023-09-06T20:34:08,786 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-16940324422883 execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:34:08,786 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-16940324422883,,1694032448347.2fb717bf2da453848b7afb361b8ed8c0.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1694032448786"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032448786"}]},"ts":"1694032448786"} 2023-09-06T20:34:08,788 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:34:08,790 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-16940324422883 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:34:08,790 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16940324422883","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032448790"}]},"ts":"1694032448790"} 2023-09-06T20:34:08,792 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16940324422883, state=ENABLING in hbase:meta 2023-09-06T20:34:08,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16940324422883, region=2fb717bf2da453848b7afb361b8ed8c0, ASSIGN}] 2023-09-06T20:34:08,798 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16940324422883, region=2fb717bf2da453848b7afb361b8ed8c0, ASSIGN 2023-09-06T20:34:08,799 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-16940324422883, region=2fb717bf2da453848b7afb361b8ed8c0, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44489,1694032433466; forceNewPlan=false, retain=false 2023-09-06T20:34:08,950 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=2fb717bf2da453848b7afb361b8ed8c0, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:08,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure 2fb717bf2da453848b7afb361b8ed8c0, server=jenkins-hbase3.apache.org,44489,1694032433466}] 2023-09-06T20:34:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:34:09,110 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-16940324422883,,1694032448347.2fb717bf2da453848b7afb361b8ed8c0. 2023-09-06T20:34:09,111 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => 2fb717bf2da453848b7afb361b8ed8c0, NAME => 'ns4:test-16940324422883,,1694032448347.2fb717bf2da453848b7afb361b8ed8c0.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:34:09,111 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:34:09,111 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16940324422883 2fb717bf2da453848b7afb361b8ed8c0 2023-09-06T20:34:09,111 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-16940324422883,,1694032448347.2fb717bf2da453848b7afb361b8ed8c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:34:09,111 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for 2fb717bf2da453848b7afb361b8ed8c0 2023-09-06T20:34:09,111 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for 2fb717bf2da453848b7afb361b8ed8c0 2023-09-06T20:34:09,113 INFO [StoreOpener-2fb717bf2da453848b7afb361b8ed8c0-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 2fb717bf2da453848b7afb361b8ed8c0 2023-09-06T20:34:09,115 INFO [StoreOpener-2fb717bf2da453848b7afb361b8ed8c0-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2fb717bf2da453848b7afb361b8ed8c0 columnFamilyName f 2023-09-06T20:34:09,115 DEBUG [StoreOpener-2fb717bf2da453848b7afb361b8ed8c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:34:09,116 INFO [StoreOpener-2fb717bf2da453848b7afb361b8ed8c0-1 {}] regionserver.HStore(324): Store=2fb717bf2da453848b7afb361b8ed8c0/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:34:09,117 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/ns4/test-16940324422883/2fb717bf2da453848b7afb361b8ed8c0 2023-09-06T20:34:09,117 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/ns4/test-16940324422883/2fb717bf2da453848b7afb361b8ed8c0 2023-09-06T20:34:09,120 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for 2fb717bf2da453848b7afb361b8ed8c0 2023-09-06T20:34:09,123 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/ns4/test-16940324422883/2fb717bf2da453848b7afb361b8ed8c0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:34:09,124 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened 2fb717bf2da453848b7afb361b8ed8c0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9783077920, jitterRate=-0.08887986838817596}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:34:09,124 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for 2fb717bf2da453848b7afb361b8ed8c0: 2023-09-06T20:34:09,126 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2556): Post open deploy tasks for ns4:test-16940324422883,,1694032448347.2fb717bf2da453848b7afb361b8ed8c0., pid=24, masterSystemTime=1694032449106 2023-09-06T20:34:09,128 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2583): Finished post open deploy task for ns4:test-16940324422883,,1694032448347.2fb717bf2da453848b7afb361b8ed8c0. 2023-09-06T20:34:09,128 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-16940324422883,,1694032448347.2fb717bf2da453848b7afb361b8ed8c0. 2023-09-06T20:34:09,129 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=2fb717bf2da453848b7afb361b8ed8c0, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:09,133 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=24, resume processing ppid=23 2023-09-06T20:34:09,133 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure 2fb717bf2da453848b7afb361b8ed8c0, server=jenkins-hbase3.apache.org,44489,1694032433466 in 178 msec 2023-09-06T20:34:09,136 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=23, resume processing ppid=22 2023-09-06T20:34:09,136 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-16940324422883, region=2fb717bf2da453848b7afb361b8ed8c0, ASSIGN in 337 msec 2023-09-06T20:34:09,136 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-16940324422883 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:34:09,137 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16940324422883","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032449136"}]},"ts":"1694032449136"} 2023-09-06T20:34:09,138 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16940324422883, state=ENABLED in hbase:meta 2023-09-06T20:34:09,141 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16940324422883 execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:34:09,143 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-16940324422883 in 793 msec 2023-09-06T20:34:09,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:34:09,460 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns4:test-16940324422883, procId: 22 completed 2023-09-06T20:34:09,460 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-16940324422883 get assigned. Timeout = 60000ms 2023-09-06T20:34:09,460 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-06T20:34:09,464 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-16940324422883 assigned to meta. Checking AM states. 2023-09-06T20:34:09,465 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-06T20:34:09,465 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-16940324422883 assigned. 2023-09-06T20:34:09,465 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1f9ce6ad to 127.0.0.1:51523 2023-09-06T20:34:09,465 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:09,497 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:34:09,510 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] hbase.ResourceChecker(147): before: backup.TestIncrementalBackupWithFailures#testIncBackupRestore[0] Thread=584, OpenFileDescriptor=667, MaxFileDescriptor=60000, SystemLoadAverage=167, ProcessCount=167, AvailableMemoryMB=3375 2023-09-06T20:34:09,510 WARN [Listener at jenkins-hbase3.apache.org/45863 {}] hbase.ResourceChecker(130): Thread=584 is superior to 500 2023-09-06T20:34:09,518 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] backup.TestIncrementalBackupWithFailures(82): create full backup image for all tables 2023-09-06T20:34:09,534 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.HMaster$13(2700): Client=jenkins//172.31.12.81 modify table test-1694032442288 from 'test-1694032442288', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1694032442288', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:34:09,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=25, state=RUNNABLE:MODIFY_TABLE_PREPARE; ModifyTableProcedure table=test-1694032442288 2023-09-06T20:34:09,996 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/.tabledesc/.tableinfo.0000000001.367 2023-09-06T20:34:09,996 INFO [PEWorker-4 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/.tabledesc/.tableinfo.0000000002.598 2023-09-06T20:34:10,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS; ReopenTableRegionsProcedure table=test-1694032442288}] 2023-09-06T20:34:10,017 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1694032442288, region=90d54d530e57af59206a5a1415c17690, REOPEN/MOVE}] 2023-09-06T20:34:10,018 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1694032442288, region=90d54d530e57af59206a5a1415c17690, REOPEN/MOVE 2023-09-06T20:34:10,019 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=90d54d530e57af59206a5a1415c17690, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:10,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; CloseRegionProcedure 90d54d530e57af59206a5a1415c17690, server=jenkins-hbase3.apache.org,44489,1694032433466}] 2023-09-06T20:34:10,181 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(119): Close 90d54d530e57af59206a5a1415c17690 2023-09-06T20:34:10,183 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1668): Closing 90d54d530e57af59206a5a1415c17690, disabling compactions & flushes 2023-09-06T20:34:10,183 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1690): Closing region test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690. 2023-09-06T20:34:10,183 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690. 2023-09-06T20:34:10,183 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1778): Acquired close lock on test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690. after waiting 0 ms 2023-09-06T20:34:10,183 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1788): Updates disabled for region test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690. 2023-09-06T20:34:10,184 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2819): Flushing 90d54d530e57af59206a5a1415c17690 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-09-06T20:34:10,272 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/90d54d530e57af59206a5a1415c17690/.tmp/f/53e5825895f94dd0954fca503bf352ed is 37, key is row10/f:q1/1694032444931/Put/seqid=0 2023-09-06T20:34:10,695 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/90d54d530e57af59206a5a1415c17690/.tmp/f/53e5825895f94dd0954fca503bf352ed 2023-09-06T20:34:10,769 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/90d54d530e57af59206a5a1415c17690/.tmp/f/53e5825895f94dd0954fca503bf352ed as hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/90d54d530e57af59206a5a1415c17690/f/53e5825895f94dd0954fca503bf352ed 2023-09-06T20:34:10,777 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/90d54d530e57af59206a5a1415c17690/f/53e5825895f94dd0954fca503bf352ed, entries=99, sequenceid=103, filesize=8.2 K 2023-09-06T20:34:10,781 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 90d54d530e57af59206a5a1415c17690 in 597ms, sequenceid=103, compaction requested=false 2023-09-06T20:34:10,791 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/90d54d530e57af59206a5a1415c17690/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-09-06T20:34:10,792 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:34:10,793 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1909): Closed test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690. 2023-09-06T20:34:10,794 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1622): Region close journal for 90d54d530e57af59206a5a1415c17690: 2023-09-06T20:34:10,794 WARN [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionServer(3735): Not adding moved region record: 90d54d530e57af59206a5a1415c17690 to self. 2023-09-06T20:34:10,797 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(163): Closed 90d54d530e57af59206a5a1415c17690 2023-09-06T20:34:10,798 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=90d54d530e57af59206a5a1415c17690, regionState=CLOSED 2023-09-06T20:34:10,802 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=28, resume processing ppid=27 2023-09-06T20:34:10,802 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=28, ppid=27, state=SUCCESS; CloseRegionProcedure 90d54d530e57af59206a5a1415c17690, server=jenkins-hbase3.apache.org,44489,1694032433466 in 779 msec 2023-09-06T20:34:10,803 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1694032442288, region=90d54d530e57af59206a5a1415c17690, REOPEN/MOVE; state=CLOSED, location=jenkins-hbase3.apache.org,44489,1694032433466; forceNewPlan=false, retain=true 2023-09-06T20:34:10,953 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=90d54d530e57af59206a5a1415c17690, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:10,956 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=29, ppid=27, state=RUNNABLE; OpenRegionProcedure 90d54d530e57af59206a5a1415c17690, server=jenkins-hbase3.apache.org,44489,1694032433466}] 2023-09-06T20:34:11,113 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(130): Open test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690. 2023-09-06T20:34:11,113 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7254): Opening region: {ENCODED => 90d54d530e57af59206a5a1415c17690, NAME => 'test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:34:11,113 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:34:11,113 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1694032442288 90d54d530e57af59206a5a1415c17690 2023-09-06T20:34:11,113 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(888): Instantiated test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:34:11,114 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7296): checking encryption for 90d54d530e57af59206a5a1415c17690 2023-09-06T20:34:11,114 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7299): checking classloading for 90d54d530e57af59206a5a1415c17690 2023-09-06T20:34:11,117 INFO [StoreOpener-90d54d530e57af59206a5a1415c17690-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 90d54d530e57af59206a5a1415c17690 2023-09-06T20:34:11,119 INFO [StoreOpener-90d54d530e57af59206a5a1415c17690-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90d54d530e57af59206a5a1415c17690 columnFamilyName f 2023-09-06T20:34:11,119 DEBUG [StoreOpener-90d54d530e57af59206a5a1415c17690-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:34:11,130 DEBUG [StoreOpener-90d54d530e57af59206a5a1415c17690-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/90d54d530e57af59206a5a1415c17690/f/53e5825895f94dd0954fca503bf352ed 2023-09-06T20:34:11,130 INFO [StoreOpener-90d54d530e57af59206a5a1415c17690-1 {}] regionserver.HStore(324): Store=90d54d530e57af59206a5a1415c17690/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:34:11,130 INFO [StoreOpener-90d54d530e57af59206a5a1415c17690-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region 90d54d530e57af59206a5a1415c17690 2023-09-06T20:34:11,132 INFO [StoreOpener-90d54d530e57af59206a5a1415c17690-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90d54d530e57af59206a5a1415c17690 columnFamilyName f3 2023-09-06T20:34:11,132 DEBUG [StoreOpener-90d54d530e57af59206a5a1415c17690-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:34:11,133 INFO [StoreOpener-90d54d530e57af59206a5a1415c17690-1 {}] regionserver.HStore(324): Store=90d54d530e57af59206a5a1415c17690/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:34:11,134 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/90d54d530e57af59206a5a1415c17690 2023-09-06T20:34:11,135 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/90d54d530e57af59206a5a1415c17690 2023-09-06T20:34:11,137 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1694032442288 descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-09-06T20:34:11,139 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1076): writing seq id for 90d54d530e57af59206a5a1415c17690 2023-09-06T20:34:11,141 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1093): Opened 90d54d530e57af59206a5a1415c17690; next sequenceid=107; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11354321920, jitterRate=0.05745363235473633}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-09-06T20:34:11,141 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(992): Region open journal for 90d54d530e57af59206a5a1415c17690: 2023-09-06T20:34:11,142 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690., pid=29, masterSystemTime=1694032451108 2023-09-06T20:34:11,145 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690. 2023-09-06T20:34:11,145 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(158): Opened test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690. 2023-09-06T20:34:11,145 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=90d54d530e57af59206a5a1415c17690, regionState=OPEN, openSeqNum=107, regionLocation=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:11,150 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=29, resume processing ppid=27 2023-09-06T20:34:11,150 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=29, ppid=27, state=SUCCESS; OpenRegionProcedure 90d54d530e57af59206a5a1415c17690, server=jenkins-hbase3.apache.org,44489,1694032433466 in 191 msec 2023-09-06T20:34:11,152 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=27, resume processing ppid=26 2023-09-06T20:34:11,152 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=test-1694032442288, region=90d54d530e57af59206a5a1415c17690, REOPEN/MOVE in 1.1330 sec 2023-09-06T20:34:11,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=26, resume processing ppid=25 2023-09-06T20:34:11,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=26, ppid=25, state=SUCCESS; ReopenTableRegionsProcedure table=test-1694032442288 in 1.1530 sec 2023-09-06T20:34:11,159 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=25, state=SUCCESS; ModifyTableProcedure table=test-1694032442288 in 1.6100 sec 2023-09-06T20:34:11,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=25 2023-09-06T20:34:11,171 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2296fa8d to 127.0.0.1:51523 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:34:11,177 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391d6e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:11,178 DEBUG [hconnection-0xfe72f5a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:11,180 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:59636, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:11,213 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:34:11,214 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:38174, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:34:11,225 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'backup'} 2023-09-06T20:34:11,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=30, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-09-06T20:34:11,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=30 2023-09-06T20:34:11,238 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:34:11,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=30, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 16 msec 2023-09-06T20:34:11,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=30 2023-09-06T20:34:11,346 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:34:11,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-09-06T20:34:11,351 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:34:11,351 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:34:11,351 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 31 2023-09-06T20:34:11,352 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:34:11,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=31 2023-09-06T20:34:11,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=31 2023-09-06T20:34:11,560 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16940324422883 2023-09-06T20:34:11,560 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-16940324422883 Metrics about Tables on a single HBase RegionServer 2023-09-06T20:34:11,561 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1694032442288 2023-09-06T20:34:11,561 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1694032442288 Metrics about Tables on a single HBase RegionServer 2023-09-06T20:34:11,562 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16940324422882 2023-09-06T20:34:11,562 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-16940324422882 Metrics about Tables on a single HBase RegionServer 2023-09-06T20:34:11,563 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16940324422881 2023-09-06T20:34:11,563 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-16940324422881 Metrics about Tables on a single HBase RegionServer 2023-09-06T20:34:11,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=31 2023-09-06T20:34:11,765 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => cc8a9b52139566dcd11c25f6d6fdfaad, NAME => 'backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928 2023-09-06T20:34:11,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=31 2023-09-06T20:34:12,177 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:34:12,178 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing cc8a9b52139566dcd11c25f6d6fdfaad, disabling compactions & flushes 2023-09-06T20:34:12,178 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:34:12,178 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:34:12,178 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. after waiting 0 ms 2023-09-06T20:34:12,178 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:34:12,178 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:34:12,178 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for cc8a9b52139566dcd11c25f6d6fdfaad: 2023-09-06T20:34:12,179 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:34:12,180 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1694032452179"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032452179"}]},"ts":"1694032452179"} 2023-09-06T20:34:12,181 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:34:12,183 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:34:12,183 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032452183"}]},"ts":"1694032452183"} 2023-09-06T20:34:12,184 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-09-06T20:34:12,189 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, ASSIGN}] 2023-09-06T20:34:12,191 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, ASSIGN 2023-09-06T20:34:12,192 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44489,1694032433466; forceNewPlan=false, retain=false 2023-09-06T20:34:12,344 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=32 updating hbase:meta row=cc8a9b52139566dcd11c25f6d6fdfaad, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:12,346 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; OpenRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad, server=jenkins-hbase3.apache.org,44489,1694032433466}] 2023-09-06T20:34:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=31 2023-09-06T20:34:12,503 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(130): Open backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:34:12,503 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7254): Opening region: {ENCODED => cc8a9b52139566dcd11c25f6d6fdfaad, NAME => 'backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:34:12,504 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:34:12,504 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:12,504 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(888): Instantiated backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:34:12,504 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7296): checking encryption for cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:12,504 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7299): checking classloading for cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:12,506 INFO [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:12,508 INFO [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc8a9b52139566dcd11c25f6d6fdfaad columnFamilyName meta 2023-09-06T20:34:12,508 DEBUG [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:34:12,510 INFO [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.HStore(324): Store=cc8a9b52139566dcd11c25f6d6fdfaad/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:34:12,510 INFO [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:12,511 INFO [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc8a9b52139566dcd11c25f6d6fdfaad columnFamilyName session 2023-09-06T20:34:12,511 DEBUG [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:34:12,512 INFO [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.HStore(324): Store=cc8a9b52139566dcd11c25f6d6fdfaad/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:34:12,513 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:12,514 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:12,515 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-09-06T20:34:12,517 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1076): writing seq id for cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:12,521 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:34:12,522 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): Opened cc8a9b52139566dcd11c25f6d6fdfaad; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9909743200, jitterRate=-0.07708324491977692}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-09-06T20:34:12,523 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(992): Region open journal for cc8a9b52139566dcd11c25f6d6fdfaad: 2023-09-06T20:34:12,524 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad., pid=33, masterSystemTime=1694032452498 2023-09-06T20:34:12,526 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:34:12,526 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(158): Opened backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:34:12,526 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=32 updating hbase:meta row=cc8a9b52139566dcd11c25f6d6fdfaad, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:12,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=33, resume processing ppid=32 2023-09-06T20:34:12,531 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=33, ppid=32, state=SUCCESS; OpenRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad, server=jenkins-hbase3.apache.org,44489,1694032433466 in 182 msec 2023-09-06T20:34:12,533 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=32, resume processing ppid=31 2023-09-06T20:34:12,533 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=32, ppid=31, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, ASSIGN in 342 msec 2023-09-06T20:34:12,534 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:34:12,534 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032452534"}]},"ts":"1694032452534"} 2023-09-06T20:34:12,536 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-09-06T20:34:12,538 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:34:12,540 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=31, state=SUCCESS; CreateTableProcedure table=backup:system in 1.1930 sec 2023-09-06T20:34:13,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=31 2023-09-06T20:34:13,460 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: backup:system, procId: 31 completed 2023-09-06T20:34:13,474 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:34:13,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-09-06T20:34:13,478 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:34:13,478 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:34:13,478 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 34 2023-09-06T20:34:13,479 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:34:13,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:34:13,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:34:13,618 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-16940324422883' 2023-09-06T20:34:13,619 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-09-06T20:34:13,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:34:13,894 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => b0d4786a840d53aeace19d3e7ef71f05, NAME => 'backup:system_bulk,,1694032453474.b0d4786a840d53aeace19d3e7ef71f05.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928 2023-09-06T20:34:14,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:34:14,308 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1694032453474.b0d4786a840d53aeace19d3e7ef71f05.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:34:14,309 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing b0d4786a840d53aeace19d3e7ef71f05, disabling compactions & flushes 2023-09-06T20:34:14,309 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1694032453474.b0d4786a840d53aeace19d3e7ef71f05. 2023-09-06T20:34:14,309 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1694032453474.b0d4786a840d53aeace19d3e7ef71f05. 2023-09-06T20:34:14,309 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1694032453474.b0d4786a840d53aeace19d3e7ef71f05. after waiting 0 ms 2023-09-06T20:34:14,309 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1694032453474.b0d4786a840d53aeace19d3e7ef71f05. 2023-09-06T20:34:14,309 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1694032453474.b0d4786a840d53aeace19d3e7ef71f05. 2023-09-06T20:34:14,309 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for b0d4786a840d53aeace19d3e7ef71f05: 2023-09-06T20:34:14,310 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:34:14,311 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1694032453474.b0d4786a840d53aeace19d3e7ef71f05.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1694032454310"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032454310"}]},"ts":"1694032454310"} 2023-09-06T20:34:14,313 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:34:14,313 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:34:14,314 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032454314"}]},"ts":"1694032454314"} 2023-09-06T20:34:14,315 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-09-06T20:34:14,320 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=b0d4786a840d53aeace19d3e7ef71f05, ASSIGN}] 2023-09-06T20:34:14,322 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=b0d4786a840d53aeace19d3e7ef71f05, ASSIGN 2023-09-06T20:34:14,323 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=b0d4786a840d53aeace19d3e7ef71f05, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44489,1694032433466; forceNewPlan=false, retain=false 2023-09-06T20:34:14,474 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=35 updating hbase:meta row=b0d4786a840d53aeace19d3e7ef71f05, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:14,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE; OpenRegionProcedure b0d4786a840d53aeace19d3e7ef71f05, server=jenkins-hbase3.apache.org,44489,1694032433466}] 2023-09-06T20:34:14,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:34:14,615 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:34:14,633 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1694032453474.b0d4786a840d53aeace19d3e7ef71f05. 2023-09-06T20:34:14,634 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7254): Opening region: {ENCODED => b0d4786a840d53aeace19d3e7ef71f05, NAME => 'backup:system_bulk,,1694032453474.b0d4786a840d53aeace19d3e7ef71f05.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:34:14,634 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:34:14,634 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk b0d4786a840d53aeace19d3e7ef71f05 2023-09-06T20:34:14,634 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1694032453474.b0d4786a840d53aeace19d3e7ef71f05.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:34:14,635 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7296): checking encryption for b0d4786a840d53aeace19d3e7ef71f05 2023-09-06T20:34:14,635 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7299): checking classloading for b0d4786a840d53aeace19d3e7ef71f05 2023-09-06T20:34:14,637 INFO [StoreOpener-b0d4786a840d53aeace19d3e7ef71f05-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region b0d4786a840d53aeace19d3e7ef71f05 2023-09-06T20:34:14,639 INFO [StoreOpener-b0d4786a840d53aeace19d3e7ef71f05-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b0d4786a840d53aeace19d3e7ef71f05 columnFamilyName meta 2023-09-06T20:34:14,639 DEBUG [StoreOpener-b0d4786a840d53aeace19d3e7ef71f05-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:34:14,640 INFO [StoreOpener-b0d4786a840d53aeace19d3e7ef71f05-1 {}] regionserver.HStore(324): Store=b0d4786a840d53aeace19d3e7ef71f05/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:34:14,640 INFO [StoreOpener-b0d4786a840d53aeace19d3e7ef71f05-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region b0d4786a840d53aeace19d3e7ef71f05 2023-09-06T20:34:14,642 INFO [StoreOpener-b0d4786a840d53aeace19d3e7ef71f05-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b0d4786a840d53aeace19d3e7ef71f05 columnFamilyName session 2023-09-06T20:34:14,642 DEBUG [StoreOpener-b0d4786a840d53aeace19d3e7ef71f05-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:34:14,643 INFO [StoreOpener-b0d4786a840d53aeace19d3e7ef71f05-1 {}] regionserver.HStore(324): Store=b0d4786a840d53aeace19d3e7ef71f05/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:34:14,644 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system_bulk/b0d4786a840d53aeace19d3e7ef71f05 2023-09-06T20:34:14,645 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system_bulk/b0d4786a840d53aeace19d3e7ef71f05 2023-09-06T20:34:14,647 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-09-06T20:34:14,649 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1076): writing seq id for b0d4786a840d53aeace19d3e7ef71f05 2023-09-06T20:34:14,657 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system_bulk/b0d4786a840d53aeace19d3e7ef71f05/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:34:14,658 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): Opened b0d4786a840d53aeace19d3e7ef71f05; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10935449280, jitterRate=0.01844307780265808}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-09-06T20:34:14,659 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(992): Region open journal for b0d4786a840d53aeace19d3e7ef71f05: 2023-09-06T20:34:14,660 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system_bulk,,1694032453474.b0d4786a840d53aeace19d3e7ef71f05., pid=36, masterSystemTime=1694032454629 2023-09-06T20:34:14,663 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system_bulk,,1694032453474.b0d4786a840d53aeace19d3e7ef71f05. 2023-09-06T20:34:14,663 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1694032453474.b0d4786a840d53aeace19d3e7ef71f05. 2023-09-06T20:34:14,664 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=35 updating hbase:meta row=b0d4786a840d53aeace19d3e7ef71f05, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:14,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=36, resume processing ppid=35 2023-09-06T20:34:14,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=36, ppid=35, state=SUCCESS; OpenRegionProcedure b0d4786a840d53aeace19d3e7ef71f05, server=jenkins-hbase3.apache.org,44489,1694032433466 in 192 msec 2023-09-06T20:34:14,673 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=35, resume processing ppid=34 2023-09-06T20:34:14,673 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=35, ppid=34, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=b0d4786a840d53aeace19d3e7ef71f05, ASSIGN in 351 msec 2023-09-06T20:34:14,674 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:34:14,674 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032454674"}]},"ts":"1694032454674"} 2023-09-06T20:34:14,676 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-09-06T20:34:14,679 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:34:14,680 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=34, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 1.2040 sec 2023-09-06T20:34:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:34:15,587 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: backup:system_bulk, procId: 34 completed 2023-09-06T20:34:15,600 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:34:15,648 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:34:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032455648 (current time:1694032455648). 2023-09-06T20:34:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:34:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:34:15,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x18d95682 to 127.0.0.1:51523 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:34:15,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5237fc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:15,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:15,667 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:59646, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:15,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x18d95682 to 127.0.0.1:51523 2023-09-06T20:34:15,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:15,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:34:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=37, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:34:15,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 37 2023-09-06T20:34:15,688 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:34:15,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=37 2023-09-06T20:34:15,694 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:34:15,697 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:34:15,710 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:34:15,713 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE; SnapshotRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad}] 2023-09-06T20:34:15,720 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=38, ppid=37, state=RUNNABLE; SnapshotRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:15,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=37 2023-09-06T20:34:15,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44489 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=38 2023-09-06T20:34:15,881 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:34:15,882 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.HRegion(2819): Flushing cc8a9b52139566dcd11c25f6d6fdfaad 2/2 column families, dataSize=45 B heapSize=632 B 2023-09-06T20:34:15,899 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/f4a9d1ee08d64003a14895000c52e7db is 49, key is activesession:/session:c/1694032455632/Put/seqid=0 2023-09-06T20:34:15,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=37 2023-09-06T20:34:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=37 2023-09-06T20:34:16,309 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/f4a9d1ee08d64003a14895000c52e7db 2023-09-06T20:34:16,318 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/f4a9d1ee08d64003a14895000c52e7db as hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/f4a9d1ee08d64003a14895000c52e7db 2023-09-06T20:34:16,328 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/f4a9d1ee08d64003a14895000c52e7db, entries=1, sequenceid=5, filesize=4.9 K 2023-09-06T20:34:16,330 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for cc8a9b52139566dcd11c25f6d6fdfaad in 448ms, sequenceid=5, compaction requested=false 2023-09-06T20:34:16,331 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.HRegion(2520): Flush status journal for cc8a9b52139566dcd11c25f6d6fdfaad: 2023-09-06T20:34:16,331 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. for snapshot_backup_system completed. 2023-09-06T20:34:16,332 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:34:16,339 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:34:16,344 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-09-06T20:34:16,344 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/f4a9d1ee08d64003a14895000c52e7db] hfiles 2023-09-06T20:34:16,344 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/f4a9d1ee08d64003a14895000c52e7db for snapshot=snapshot_backup_system 2023-09-06T20:34:16,759 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:34:16,763 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=38 2023-09-06T20:34:16,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.HMaster(3995): Remote procedure done, pid=38 2023-09-06T20:34:16,767 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:16,768 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=38, ppid=37, state=RUNNABLE; SnapshotRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:16,771 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=38, resume processing ppid=37 2023-09-06T20:34:16,771 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:34:16,771 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=38, ppid=37, state=SUCCESS; SnapshotRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad in 1.0560 sec 2023-09-06T20:34:16,773 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:34:16,776 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:34:16,776 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:34:16,779 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:34:16,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=37 2023-09-06T20:34:17,205 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:34:17,219 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:34:17,220 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:34:17,222 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:34:17,222 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 37 2023-09-06T20:34:17,224 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=37, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.5420 sec 2023-09-06T20:34:17,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=37 2023-09-06T20:34:17,802 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: backup:system, procId: 37 completed 2023-09-06T20:34:17,803 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(115): Backup backup_1694032451196 started at 1694032457802. 2023-09-06T20:34:17,826 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(119): Backup session backup_1694032451196 has been started. 2023-09-06T20:34:17,829 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-09-06T20:34:17,839 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-09-06T20:34:17,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-09-06T20:34:17,848 INFO [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-09-06T20:34:17,848 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-09-06T20:34:17,848 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-09-06T20:34:17,848 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-09-06T20:34:17,849 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-09-06T20:34:17,849 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:34:17,850 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44489-0x100eefce21b0001, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-09-06T20:34:17,850 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:17,851 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:34:17,851 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:34:17,851 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:17,851 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-09-06T20:34:17,851 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-09-06T20:34:17,851 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44489-0x100eefce21b0001, quorum=127.0.0.1:51523, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-09-06T20:34:17,852 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-09-06T20:34:17,852 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:34:17,852 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-09-06T20:34:17,854 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-09-06T20:34:17,854 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-09-06T20:34:17,854 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-09-06T20:34:17,854 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-09-06T20:34:17,855 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-09-06T20:34:17,855 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-09-06T20:34:17,855 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,44489,1694032433466' joining acquired barrier for procedure (rolllog) in zk 2023-09-06T20:34:17,857 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:17,857 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-09-06T20:34:17,857 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:17,857 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:34:17,857 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:34:17,857 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44489-0x100eefce21b0001, quorum=127.0.0.1:51523, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-09-06T20:34:17,857 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-09-06T20:34:17,857 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:34:17,858 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:34:17,858 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:17,858 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:17,859 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:34:17,859 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,44489,1694032433466' joining acquired barrier for procedure 'rolllog' on coordinator 2023-09-06T20:34:17,859 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@67682e2c[Count = 0] remaining members to acquire global barrier 2023-09-06T20:34:17,859 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-09-06T20:34:17,859 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-09-06T20:34:17,861 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44489-0x100eefce21b0001, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-09-06T20:34:17,861 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-09-06T20:34:17,861 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-09-06T20:34:17,861 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-09-06T20:34:17,861 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:17,861 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-09-06T20:34:17,862 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-09-06T20:34:17,862 DEBUG [rs(jenkins-hbase3.apache.org,44489,1694032433466)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:17,863 INFO [rs(jenkins-hbase3.apache.org,44489,1694032433466)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1694032435590 highest: 1694032437629 on jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:17,863 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0:(num 1694032435590) roll requested 2023-09-06T20:34:17,874 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032457863, exclude list is [], retry=0 2023-09-06T20:34:17,879 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK] 2023-09-06T20:34:17,883 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032435590 with entries=18, filesize=3.96 KB; new WAL /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032457863 2023-09-06T20:34:17,884 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK]] 2023-09-06T20:34:17,884 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032435590 is not closed yet, will try archiving it next time 2023-09-06T20:34:17,884 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1:(num 1694032437629) roll requested 2023-09-06T20:34:17,897 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032457884, exclude list is [], retry=0 2023-09-06T20:34:17,900 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK] 2023-09-06T20:34:17,902 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032437629 with entries=11, filesize=2.04 KB; new WAL /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032457884 2023-09-06T20:34:17,903 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK]] 2023-09-06T20:34:17,903 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032437629 is not closed yet, will try archiving it next time 2023-09-06T20:34:17,903 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta:.meta(num 1694032435949) roll requested 2023-09-06T20:34:17,915 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta.1694032457903.meta, exclude list is [], retry=0 2023-09-06T20:34:17,918 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK] 2023-09-06T20:34:17,921 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta.1694032435949.meta with entries=40, filesize=12.94 KB; new WAL /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta.1694032457903.meta 2023-09-06T20:34:17,921 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK]] 2023-09-06T20:34:17,921 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta.1694032435949.meta is not closed yet, will try archiving it next time 2023-09-06T20:34:17,923 DEBUG [rs(jenkins-hbase3.apache.org,44489,1694032433466)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2023-09-06T20:34:17,923 INFO [rs(jenkins-hbase3.apache.org,44489,1694032433466)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1694032457863 on jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:17,924 DEBUG [rs(jenkins-hbase3.apache.org,44489,1694032433466)-backup-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:34:17,925 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:38188, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-09-06T20:34:17,944 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-09-06T20:34:17,944 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,44489,1694032433466' in zk 2023-09-06T20:34:17,948 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:17,948 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-09-06T20:34:17,948 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:34:17,948 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-09-06T20:34:17,948 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:17,948 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:34:17,948 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:34:17,949 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:34:17,949 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:34:17,949 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:17,950 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:17,950 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:34:17,950 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:17,951 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:17,952 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,44489,1694032433466': 2023-09-06T20:34:17,952 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,44489,1694032433466' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-09-06T20:34:17,952 INFO [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-09-06T20:34:17,952 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-09-06T20:34:17,952 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-09-06T20:34:17,952 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-09-06T20:34:17,952 INFO [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-09-06T20:34:17,955 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-09-06T20:34:17,955 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44489-0x100eefce21b0001, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-09-06T20:34:17,955 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44489-0x100eefce21b0001, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-09-06T20:34:17,955 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-09-06T20:34:17,955 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-09-06T20:34:17,955 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:34:17,955 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:34:17,955 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:34:17,955 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:34:17,955 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:34:17,955 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:34:17,955 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:17,956 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:17,956 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:34:17,956 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:34:17,957 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:17,957 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:17,957 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:17,958 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:34:17,958 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:17,958 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:17,965 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:17,966 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44489-0x100eefce21b0001, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-09-06T20:34:17,966 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-09-06T20:34:17,966 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-09-06T20:34:17,966 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44489-0x100eefce21b0001, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-09-06T20:34:17,966 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:34:17,966 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:17,966 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:34:17,966 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-09-06T20:34:17,966 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-09-06T20:34:17,966 DEBUG [Listener at localhost/46585-EventThread {}] zookeeper.ZKWatcher(604): master:45185-0x100eefce21b0000, quorum=127.0.0.1:51523, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-09-06T20:34:17,966 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:34:17,966 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-09-06T20:34:17,967 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-09-06T20:34:17,967 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:34:17,967 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:34:17,969 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] client.HBaseAdmin(2838): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-09-06T20:34:17,969 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] client.HBaseAdmin(2846): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-09-06T20:34:18,069 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] client.HBaseAdmin(2852): Getting current status of procedure from master... 2023-09-06T20:34:18,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-09-06T20:34:18,079 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1694032458078_default_test-1694032442288 table=test-1694032442288 type=FLUSH ttl=0 } 2023-09-06T20:34:18,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032458079 (current time:1694032458079). 2023-09-06T20:34:18,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:34:18,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1694032458078_default_test-1694032442288 VERSION not specified, setting to 2 2023-09-06T20:34:18,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x46305b03 to 127.0.0.1:51523 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:34:18,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a74b73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:18,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:18,089 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:59662, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:18,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x46305b03 to 127.0.0.1:51523 2023-09-06T20:34:18,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:18,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:34:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=39, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1694032458078_default_test-1694032442288 table=test-1694032442288 type=FLUSH ttl=0 } 2023-09-06T20:34:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1694032458078_default_test-1694032442288 table=test-1694032442288 type=FLUSH ttl=0 }, snapshot procedure id = 39 2023-09-06T20:34:18,094 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1694032458078_default_test-1694032442288 table=test-1694032442288 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:34:18,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-09-06T20:34:18,095 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1694032458078_default_test-1694032442288 table=test-1694032442288 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:34:18,097 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1694032458078_default_test-1694032442288 table=test-1694032442288 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:34:18,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-09-06T20:34:18,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-09-06T20:34:18,506 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1694032458078_default_test-1694032442288 table=test-1694032442288 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:34:18,507 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE; SnapshotRegionProcedure 90d54d530e57af59206a5a1415c17690}] 2023-09-06T20:34:18,509 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=40, ppid=39, state=RUNNABLE; SnapshotRegionProcedure 90d54d530e57af59206a5a1415c17690 2023-09-06T20:34:18,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44489 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=40 2023-09-06T20:34:18,661 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690. 2023-09-06T20:34:18,662 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2819): Flushing 90d54d530e57af59206a5a1415c17690 2/2 column families, dataSize=324 B heapSize=1.25 KB 2023-09-06T20:34:18,678 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/90d54d530e57af59206a5a1415c17690/.tmp/f3/0c7ddabb35ec40e28c445405f153e802 is 58, key is row-test-1694032442288-3-0/f3:q1/1694032451184/Put/seqid=0 2023-09-06T20:34:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-09-06T20:34:19,090 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=324 B at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/90d54d530e57af59206a5a1415c17690/.tmp/f3/0c7ddabb35ec40e28c445405f153e802 2023-09-06T20:34:19,099 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/90d54d530e57af59206a5a1415c17690/.tmp/f3/0c7ddabb35ec40e28c445405f153e802 as hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/90d54d530e57af59206a5a1415c17690/f3/0c7ddabb35ec40e28c445405f153e802 2023-09-06T20:34:19,106 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/90d54d530e57af59206a5a1415c17690/f3/0c7ddabb35ec40e28c445405f153e802, entries=6, sequenceid=115, filesize=5.2 K 2023-09-06T20:34:19,108 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(3022): Finished flush of dataSize ~324 B/324, heapSize ~1008 B/1008, currentSize=0 B/0 for 90d54d530e57af59206a5a1415c17690 in 446ms, sequenceid=115, compaction requested=false 2023-09-06T20:34:19,108 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2520): Flush status journal for 90d54d530e57af59206a5a1415c17690: 2023-09-06T20:34:19,108 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690. for snapshot_1694032458078_default_test-1694032442288 completed. 2023-09-06T20:34:19,109 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(240): Storing 'test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690.' region-info for snapshot=snapshot_1694032458078_default_test-1694032442288 2023-09-06T20:34:19,109 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:34:19,109 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/90d54d530e57af59206a5a1415c17690/f/53e5825895f94dd0954fca503bf352ed] hfiles 2023-09-06T20:34:19,109 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/90d54d530e57af59206a5a1415c17690/f/53e5825895f94dd0954fca503bf352ed for snapshot=snapshot_1694032458078_default_test-1694032442288 2023-09-06T20:34:19,109 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/90d54d530e57af59206a5a1415c17690/f3/0c7ddabb35ec40e28c445405f153e802] hfiles 2023-09-06T20:34:19,109 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/90d54d530e57af59206a5a1415c17690/f3/0c7ddabb35ec40e28c445405f153e802 for snapshot=snapshot_1694032458078_default_test-1694032442288 2023-09-06T20:34:19,118 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690. 2023-09-06T20:34:19,118 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=40 2023-09-06T20:34:19,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.HMaster(3995): Remote procedure done, pid=40 2023-09-06T20:34:19,119 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1694032458078_default_test-1694032442288 on region 90d54d530e57af59206a5a1415c17690 2023-09-06T20:34:19,120 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=40, ppid=39, state=RUNNABLE; SnapshotRegionProcedure 90d54d530e57af59206a5a1415c17690 2023-09-06T20:34:19,123 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=40, resume processing ppid=39 2023-09-06T20:34:19,123 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1694032458078_default_test-1694032442288 table=test-1694032442288 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:34:19,123 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=40, ppid=39, state=SUCCESS; SnapshotRegionProcedure 90d54d530e57af59206a5a1415c17690 in 614 msec 2023-09-06T20:34:19,124 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1694032458078_default_test-1694032442288 table=test-1694032442288 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:34:19,124 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1694032458078_default_test-1694032442288 table=test-1694032442288 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:34:19,124 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1694032458078_default_test-1694032442288 2023-09-06T20:34:19,125 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/.tmp/snapshot_1694032458078_default_test-1694032442288 2023-09-06T20:34:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-09-06T20:34:19,543 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1694032458078_default_test-1694032442288 table=test-1694032442288 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:34:19,551 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1694032458078_default_test-1694032442288 table=test-1694032442288 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:34:19,551 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/.tmp/snapshot_1694032458078_default_test-1694032442288 to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/snapshot_1694032458078_default_test-1694032442288 2023-09-06T20:34:19,553 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1694032458078_default_test-1694032442288 table=test-1694032442288 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:34:19,553 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1694032458078_default_test-1694032442288 table=test-1694032442288 type=FLUSH ttl=0 }, snapshot procedure id = 39 2023-09-06T20:34:19,554 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=39, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1694032458078_default_test-1694032442288 table=test-1694032442288 type=FLUSH ttl=0 } in 1.4620 sec 2023-09-06T20:34:19,620 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-09-06T20:34:20,139 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:34:20,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-09-06T20:34:20,201 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: default:test-1694032442288, procId: 39 completed 2023-09-06T20:34:20,202 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1694032460201_ns2_test-16940324422881 table=ns2:test-16940324422881 type=FLUSH ttl=0 } 2023-09-06T20:34:20,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032460202 (current time:1694032460202). 2023-09-06T20:34:20,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:34:20,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1694032460201_ns2_test-16940324422881 VERSION not specified, setting to 2 2023-09-06T20:34:20,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7ac65cd8 to 127.0.0.1:51523 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:34:20,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e6ae8c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:20,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:20,216 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50676, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:20,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7ac65cd8 to 127.0.0.1:51523 2023-09-06T20:34:20,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:20,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:34:20,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=41, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1694032460201_ns2_test-16940324422881 table=ns2:test-16940324422881 type=FLUSH ttl=0 } 2023-09-06T20:34:20,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1694032460201_ns2_test-16940324422881 table=ns2:test-16940324422881 type=FLUSH ttl=0 }, snapshot procedure id = 41 2023-09-06T20:34:20,221 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1694032460201_ns2_test-16940324422881 table=ns2:test-16940324422881 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:34:20,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=41 2023-09-06T20:34:20,222 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1694032460201_ns2_test-16940324422881 table=ns2:test-16940324422881 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:34:20,225 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1694032460201_ns2_test-16940324422881 table=ns2:test-16940324422881 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:34:20,234 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1694032460201_ns2_test-16940324422881 table=ns2:test-16940324422881 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:34:20,234 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE; SnapshotRegionProcedure 95242ef863d74dd33bf9cf6336515789}] 2023-09-06T20:34:20,235 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=42, ppid=41, state=RUNNABLE; SnapshotRegionProcedure 95242ef863d74dd33bf9cf6336515789 2023-09-06T20:34:20,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=41 2023-09-06T20:34:20,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44489 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=42 2023-09-06T20:34:20,388 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16940324422881,,1694032445019.95242ef863d74dd33bf9cf6336515789. 2023-09-06T20:34:20,389 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.HRegion(2819): Flushing 95242ef863d74dd33bf9cf6336515789 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-09-06T20:34:20,408 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/ns2/test-16940324422881/95242ef863d74dd33bf9cf6336515789/.tmp/f/70c2171bfd0c4daa9c6babd39d04f12f is 37, key is row10/f:q1/1694032446145/Put/seqid=0 2023-09-06T20:34:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=41 2023-09-06T20:34:20,818 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/ns2/test-16940324422881/95242ef863d74dd33bf9cf6336515789/.tmp/f/70c2171bfd0c4daa9c6babd39d04f12f 2023-09-06T20:34:20,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=41 2023-09-06T20:34:20,827 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/ns2/test-16940324422881/95242ef863d74dd33bf9cf6336515789/.tmp/f/70c2171bfd0c4daa9c6babd39d04f12f as hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/ns2/test-16940324422881/95242ef863d74dd33bf9cf6336515789/f/70c2171bfd0c4daa9c6babd39d04f12f 2023-09-06T20:34:20,833 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/ns2/test-16940324422881/95242ef863d74dd33bf9cf6336515789/f/70c2171bfd0c4daa9c6babd39d04f12f, entries=99, sequenceid=103, filesize=8.2 K 2023-09-06T20:34:20,835 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 95242ef863d74dd33bf9cf6336515789 in 445ms, sequenceid=103, compaction requested=false 2023-09-06T20:34:20,836 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.HRegion(2520): Flush status journal for 95242ef863d74dd33bf9cf6336515789: 2023-09-06T20:34:20,836 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16940324422881,,1694032445019.95242ef863d74dd33bf9cf6336515789. for snapshot_1694032460201_ns2_test-16940324422881 completed. 2023-09-06T20:34:20,836 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16940324422881,,1694032445019.95242ef863d74dd33bf9cf6336515789.' region-info for snapshot=snapshot_1694032460201_ns2_test-16940324422881 2023-09-06T20:34:20,836 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:34:20,836 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/ns2/test-16940324422881/95242ef863d74dd33bf9cf6336515789/f/70c2171bfd0c4daa9c6babd39d04f12f] hfiles 2023-09-06T20:34:20,836 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/ns2/test-16940324422881/95242ef863d74dd33bf9cf6336515789/f/70c2171bfd0c4daa9c6babd39d04f12f for snapshot=snapshot_1694032460201_ns2_test-16940324422881 2023-09-06T20:34:21,245 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16940324422881,,1694032445019.95242ef863d74dd33bf9cf6336515789. 2023-09-06T20:34:21,245 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=42 2023-09-06T20:34:21,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.HMaster(3995): Remote procedure done, pid=42 2023-09-06T20:34:21,246 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1694032460201_ns2_test-16940324422881 on region 95242ef863d74dd33bf9cf6336515789 2023-09-06T20:34:21,248 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=42, ppid=41, state=RUNNABLE; SnapshotRegionProcedure 95242ef863d74dd33bf9cf6336515789 2023-09-06T20:34:21,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=42, resume processing ppid=41 2023-09-06T20:34:21,251 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1694032460201_ns2_test-16940324422881 table=ns2:test-16940324422881 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:34:21,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=42, ppid=41, state=SUCCESS; SnapshotRegionProcedure 95242ef863d74dd33bf9cf6336515789 in 1.0150 sec 2023-09-06T20:34:21,252 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1694032460201_ns2_test-16940324422881 table=ns2:test-16940324422881 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:34:21,253 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1694032460201_ns2_test-16940324422881 table=ns2:test-16940324422881 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:34:21,253 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1694032460201_ns2_test-16940324422881 2023-09-06T20:34:21,254 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/.tmp/snapshot_1694032460201_ns2_test-16940324422881 2023-09-06T20:34:21,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=41 2023-09-06T20:34:21,560 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-09-06T20:34:21,560 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-09-06T20:34:21,561 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-09-06T20:34:21,561 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-09-06T20:34:21,671 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1694032460201_ns2_test-16940324422881 table=ns2:test-16940324422881 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:34:21,679 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1694032460201_ns2_test-16940324422881 table=ns2:test-16940324422881 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:34:21,679 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/.tmp/snapshot_1694032460201_ns2_test-16940324422881 to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/snapshot_1694032460201_ns2_test-16940324422881 2023-09-06T20:34:21,681 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1694032460201_ns2_test-16940324422881 table=ns2:test-16940324422881 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:34:21,681 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1694032460201_ns2_test-16940324422881 table=ns2:test-16940324422881 type=FLUSH ttl=0 }, snapshot procedure id = 41 2023-09-06T20:34:21,682 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=41, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1694032460201_ns2_test-16940324422881 table=ns2:test-16940324422881 type=FLUSH ttl=0 } in 1.4630 sec 2023-09-06T20:34:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=41 2023-09-06T20:34:22,328 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: ns2:test-16940324422881, procId: 41 completed 2023-09-06T20:34:22,328 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1694032451196 2023-09-06T20:34:22,329 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-09-06T20:34:22,330 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-09-06T20:34:22,333 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1694032451196_ns2:test-16940324422881 2023-09-06T20:34:22,333 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1694032460201_ns2_test-16940324422881 to hdfs://localhost:38135/backupUT/backup_1694032451196/ns2/test-16940324422881/ 2023-09-06T20:34:22,342 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-09-06T20:34:22,374 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:38135, inputRoot=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928 2023-09-06T20:34:22,374 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,45185,1694032430454_-604240450_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:38135/backupUT/backup_1694032451196/ns2/test-16940324422881, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:38135/backupUT/backup_1694032451196/ns2/test-16940324422881/.hbase-snapshot/.tmp/snapshot_1694032460201_ns2_test-16940324422881 2023-09-06T20:34:22,374 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:38135, inputRoot=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928, snapshotDir=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/snapshot_1694032460201_ns2_test-16940324422881. 2023-09-06T20:34:22,385 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/snapshot_1694032460201_ns2_test-16940324422881 to hdfs://localhost:38135/backupUT/backup_1694032451196/ns2/test-16940324422881/.hbase-snapshot/.tmp/snapshot_1694032460201_ns2_test-16940324422881 2023-09-06T20:34:22,819 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:22,820 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:22,820 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:22,821 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:23,798 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/hadoop-410939079903943071.jar 2023-09-06T20:34:23,798 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:23,799 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:23,799 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:23,800 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:23,801 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:23,801 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:23,802 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:23,803 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:23,803 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:23,805 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:34:23,806 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:34:23,806 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:34:23,807 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:34:23,807 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:34:23,808 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:34:23,808 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:34:23,808 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:34:23,809 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:34:23,809 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:34:23,810 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:34:23,810 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:34:23,814 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:23,815 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:23,815 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:34:23,816 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:23,816 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:23,817 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:34:23,817 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:34:23,876 WARN [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-09-06T20:34:23,898 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1694032460201_ns2_test-16940324422881' hfile list 2023-09-06T20:34:23,907 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-09-06T20:34:24,133 WARN [Listener at jenkins-hbase3.apache.org/45863 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/mapreduce.cluster.local.dir/1694032464082/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:34:24,133 WARN [Listener at jenkins-hbase3.apache.org/45863 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/mapreduce.cluster.local.dir/1694032464082/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:34:24,291 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-09-06T20:34:24,320 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/ns2/test-16940324422881/95242ef863d74dd33bf9cf6336515789/f/70c2171bfd0c4daa9c6babd39d04f12f output=hdfs://localhost:38135/backupUT/backup_1694032451196/ns2/test-16940324422881/archive/data/ns2/test-16940324422881/95242ef863d74dd33bf9cf6336515789/f/70c2171bfd0c4daa9c6babd39d04f12f 2023-09-06T20:34:24,321 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-09-06T20:34:25,191 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-09-06T20:34:25,192 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-09-06T20:34:25,200 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1694032460201_ns2_test-16940324422881 2023-09-06T20:34:25,200 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1694032460201_ns2_test-16940324422881 finished. 2023-09-06T20:34:25,200 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1694032451196_test-1694032442288 2023-09-06T20:34:25,200 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1694032458078_default_test-1694032442288 to hdfs://localhost:38135/backupUT/backup_1694032451196/default/test-1694032442288/ 2023-09-06T20:34:25,200 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-09-06T20:34:25,223 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:38135, inputRoot=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928 2023-09-06T20:34:25,223 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,45185,1694032430454_1476490409_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:38135/backupUT/backup_1694032451196/default/test-1694032442288, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:38135/backupUT/backup_1694032451196/default/test-1694032442288/.hbase-snapshot/.tmp/snapshot_1694032458078_default_test-1694032442288 2023-09-06T20:34:25,223 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:38135, inputRoot=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928, snapshotDir=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/snapshot_1694032458078_default_test-1694032442288. 2023-09-06T20:34:25,233 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/snapshot_1694032458078_default_test-1694032442288 to hdfs://localhost:38135/backupUT/backup_1694032451196/default/test-1694032442288/.hbase-snapshot/.tmp/snapshot_1694032458078_default_test-1694032442288 2023-09-06T20:34:25,647 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:25,647 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:25,648 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:25,648 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:26,649 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/hadoop-7290360338388903052.jar 2023-09-06T20:34:26,649 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:26,650 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:26,650 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:26,651 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:26,651 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:26,652 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:26,652 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:26,652 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:26,653 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:26,654 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:34:26,654 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:34:26,655 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:34:26,655 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:34:26,656 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:34:26,656 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:34:26,656 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:34:26,657 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:34:26,657 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:34:26,658 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:34:26,658 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:34:26,659 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:34:26,659 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:26,660 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:26,661 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:34:26,661 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:26,661 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:34:26,662 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:34:26,663 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:34:26,690 WARN [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-09-06T20:34:26,707 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1694032458078_default_test-1694032442288' hfile list 2023-09-06T20:34:26,710 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] snapshot.ExportSnapshot(674): export split=0 size=13.4 K 2023-09-06T20:34:26,784 WARN [Listener at jenkins-hbase3.apache.org/45863 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/mapreduce.cluster.local.dir/1694032466750/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:34:26,784 WARN [Listener at jenkins-hbase3.apache.org/45863 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/mapreduce.cluster.local.dir/1694032466750/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:34:26,839 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-09-06T20:34:26,890 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/90d54d530e57af59206a5a1415c17690/f/53e5825895f94dd0954fca503bf352ed output=hdfs://localhost:38135/backupUT/backup_1694032451196/default/test-1694032442288/archive/data/default/test-1694032442288/90d54d530e57af59206a5a1415c17690/f/53e5825895f94dd0954fca503bf352ed 2023-09-06T20:34:26,890 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 4.004M/sec 2023-09-06T20:34:27,323 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/90d54d530e57af59206a5a1415c17690/f3/0c7ddabb35ec40e28c445405f153e802 output=hdfs://localhost:38135/backupUT/backup_1694032451196/default/test-1694032442288/archive/data/default/test-1694032442288/90d54d530e57af59206a5a1415c17690/f3/0c7ddabb35ec40e28c445405f153e802 2023-09-06T20:34:27,324 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=5370 (5.2 K) time=0sec 5.121M/sec 2023-09-06T20:34:27,821 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-09-06T20:34:27,822 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-09-06T20:34:27,831 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1694032458078_default_test-1694032442288 2023-09-06T20:34:27,831 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1694032458078_default_test-1694032442288 finished. 2023-09-06T20:34:27,832 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(1014): ns2:test-16940324422881 2023-09-06T20:34:27,832 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(1014): test-1694032442288 2023-09-06T20:34:27,912 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1694032451196 2023-09-06T20:34:27,912 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-09-06T20:34:28,084 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-09-06T20:34:28,088 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:36542, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-09-06T20:34:28,330 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:38135/backupUT/backup_1694032451196/.backup.manifest 2023-09-06T20:34:28,330 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1694032451196 2023-09-06T20:34:28,330 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-09-06T20:34:28,736 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:38135/backupUT/backup_1694032451196/.backup.manifest 2023-09-06T20:34:28,737 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(358): Backup backup_1694032451196 finished: type=FULL,tablelist=ns2:test-16940324422881;test-1694032442288,targetRootDir=hdfs://localhost:38135/backupUT,startts=1694032457802,completets=1694032467908,bytescopied=0 2023-09-06T20:34:28,737 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-09-06T20:34:28,737 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1694032460201_ns2_test-16940324422881 2023-09-06T20:34:28,747 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1694032460201_ns2_test-16940324422881" 2023-09-06T20:34:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1694032460201_ns2_test-16940324422881 2023-09-06T20:34:28,753 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1694032460201_ns2_test-16940324422881 for backup backup_1694032451196 succeeded. 2023-09-06T20:34:28,753 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1694032458078_default_test-1694032442288 2023-09-06T20:34:28,754 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1694032458078_default_test-1694032442288" 2023-09-06T20:34:28,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1694032458078_default_test-1694032442288 2023-09-06T20:34:28,757 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1694032458078_default_test-1694032442288 for backup backup_1694032451196 succeeded. 2023-09-06T20:34:28,758 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:34:28,770 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:34:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:34:28,773 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:34:28,781 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:34:28,798 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(376): Backup backup_1694032451196 completed. 2023-09-06T20:34:28,925 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] backup.TestIncrementalBackupWithFailures(104): writing 99 rows to test-1694032442288 2023-09-06T20:34:28,940 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] backup.TestIncrementalBackupWithFailures(108): written 99 rows to test-1694032442288 2023-09-06T20:34:28,958 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] backup.TestIncrementalBackupWithFailures(120): written 5 rows to ns2:test-16940324422881 2023-09-06T20:34:28,959 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] backup.TestIncrementalBackupWithFailures(136): Running stage 0 2023-09-06T20:34:29,003 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@633825ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:29,004 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:34:29,005 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:36556, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:34:29,009 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:29,011 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50692, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:29,067 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:34:29,071 ERROR [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(228): Unexpected Exception : Failed stage stage_0 in testing java.io.IOException: Failed stage stage_0 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:123) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:34:29,072 ERROR [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(240): BackupId=backup_1694032469038,startts=0,failedts=1694032469071,failedphase=REQUEST,failedmessage=Failed stage stage_0 in testing 2023-09-06T20:34:29,072 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-09-06T20:34:29,074 WARN [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(1530): Could not restore backup system table. Snapshot snapshot_backup_system does not exists. 2023-09-06T20:34:29,074 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:34:29,076 ERROR [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(1559): Snapshot snapshot_backup_system does not exists 2023-09-06T20:34:29,076 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-09-06T20:34:29,077 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:34:29,079 ERROR [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(246): Backup backup_1694032469038 failed. Backup session finished. Status: FAILURE 2023-09-06T20:34:29,080 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:29,080 ERROR [Listener at jenkins-hbase3.apache.org/45863 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_0 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:168) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_0 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:123) ~[test-classes/:?] ... 41 more 2023-09-06T20:34:29,085 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] backup.TestIncrementalBackupWithFailures(136): Running stage 1 2023-09-06T20:34:29,104 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@380bb30b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:29,105 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:34:29,106 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:36566, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:34:29,110 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:29,112 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50706, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:29,160 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:34:29,164 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:34:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032469164 (current time:1694032469164). 2023-09-06T20:34:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:34:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:34:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77c4f09e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:29,176 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50720, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:34:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=43, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:34:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 43 2023-09-06T20:34:29,180 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:34:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=43 2023-09-06T20:34:29,181 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:34:29,184 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:34:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=43 2023-09-06T20:34:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=43 2023-09-06T20:34:29,591 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:34:29,592 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE; SnapshotRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad}] 2023-09-06T20:34:29,594 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=44, ppid=43, state=RUNNABLE; SnapshotRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:29,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44489 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=44 2023-09-06T20:34:29,745 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:34:29,746 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegion(2819): Flushing cc8a9b52139566dcd11c25f6d6fdfaad 2/2 column families, dataSize=1.83 KB heapSize=3.35 KB 2023-09-06T20:34:29,761 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/meta/89e49a43114a459a838964cbd810a62a is 171, key is trslm:hdfs://localhost:38135/backupUT\x00ns2:test-16940324422881/meta:log-roll-map/1694032467895/Put/seqid=0 2023-09-06T20:34:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=43 2023-09-06T20:34:30,166 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=785 B at sequenceid=20 (bloomFilter=true), to=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/meta/89e49a43114a459a838964cbd810a62a 2023-09-06T20:34:30,177 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/ae2e5dd8863b45f491fb822de06cae0f is 459, key is session:backup_1694032451196/session:context/1694032468780/Put/seqid=0 2023-09-06T20:34:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=43 2023-09-06T20:34:30,582 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.07 KB at sequenceid=20 (bloomFilter=true), to=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/ae2e5dd8863b45f491fb822de06cae0f 2023-09-06T20:34:30,590 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/meta/89e49a43114a459a838964cbd810a62a as hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/89e49a43114a459a838964cbd810a62a 2023-09-06T20:34:30,596 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/89e49a43114a459a838964cbd810a62a, entries=6, sequenceid=20, filesize=5.8 K 2023-09-06T20:34:30,597 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/ae2e5dd8863b45f491fb822de06cae0f as hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/ae2e5dd8863b45f491fb822de06cae0f 2023-09-06T20:34:30,604 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/ae2e5dd8863b45f491fb822de06cae0f, entries=3, sequenceid=20, filesize=5.7 K 2023-09-06T20:34:30,605 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegion(3022): Finished flush of dataSize ~1.83 KB/1879, heapSize ~3.32 KB/3400, currentSize=0 B/0 for cc8a9b52139566dcd11c25f6d6fdfaad in 859ms, sequenceid=20, compaction requested=false 2023-09-06T20:34:30,605 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegion(2520): Flush status journal for cc8a9b52139566dcd11c25f6d6fdfaad: 2023-09-06T20:34:30,605 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. for snapshot_backup_system completed. 2023-09-06T20:34:30,605 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:34:30,605 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:34:30,605 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/89e49a43114a459a838964cbd810a62a] hfiles 2023-09-06T20:34:30,605 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/89e49a43114a459a838964cbd810a62a for snapshot=snapshot_backup_system 2023-09-06T20:34:30,605 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/f4a9d1ee08d64003a14895000c52e7db, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/ae2e5dd8863b45f491fb822de06cae0f] hfiles 2023-09-06T20:34:30,605 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/f4a9d1ee08d64003a14895000c52e7db for snapshot=snapshot_backup_system 2023-09-06T20:34:30,605 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/ae2e5dd8863b45f491fb822de06cae0f for snapshot=snapshot_backup_system 2023-09-06T20:34:31,011 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:34:31,011 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=44 2023-09-06T20:34:31,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.HMaster(3995): Remote procedure done, pid=44 2023-09-06T20:34:31,012 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:31,013 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=44, ppid=43, state=RUNNABLE; SnapshotRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:31,015 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=44, resume processing ppid=43 2023-09-06T20:34:31,015 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:34:31,015 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=44, ppid=43, state=SUCCESS; SnapshotRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad in 1.4220 sec 2023-09-06T20:34:31,016 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:34:31,017 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:34:31,017 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:34:31,017 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:34:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=43 2023-09-06T20:34:31,428 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:34:31,436 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:34:31,437 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:34:31,441 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:34:31,441 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 43 2023-09-06T20:34:31,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=43, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2630 sec 2023-09-06T20:34:33,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=43 2023-09-06T20:34:33,289 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(115): Backup backup_1694032469137 started at 1694032473289. 2023-09-06T20:34:33,291 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(119): Backup session backup_1694032469137 has been started. 2023-09-06T20:34:33,292 ERROR [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(228): Unexpected Exception : Failed stage stage_1 in testing java.io.IOException: Failed stage stage_1 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:34:33,292 ERROR [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(240): BackupId=backup_1694032469137,startts=1694032473289,failedts=1694032473292,failedphase=REQUEST,failedmessage=Failed stage stage_1 in testing 2023-09-06T20:34:33,292 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-09-06T20:34:33,304 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable backup:system 2023-09-06T20:34:33,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=45, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-09-06T20:34:33,316 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032473316"}]},"ts":"1694032473316"} 2023-09-06T20:34:33,318 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-09-06T20:34:33,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=45 2023-09-06T20:34:33,320 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-09-06T20:34:33,321 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, UNASSIGN}] 2023-09-06T20:34:33,323 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, UNASSIGN 2023-09-06T20:34:33,323 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=cc8a9b52139566dcd11c25f6d6fdfaad, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:33,325 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE; CloseRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad, server=jenkins-hbase3.apache.org,44489,1694032433466}] 2023-09-06T20:34:33,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=45 2023-09-06T20:34:33,477 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] handler.UnassignRegionHandler(119): Close cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:33,477 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1668): Closing cc8a9b52139566dcd11c25f6d6fdfaad, disabling compactions & flushes 2023-09-06T20:34:33,477 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1690): Closing region backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:34:33,477 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:34:33,477 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. after waiting 0 ms 2023-09-06T20:34:33,477 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:34:33,477 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(2819): Flushing cc8a9b52139566dcd11c25f6d6fdfaad 2/2 column families, dataSize=214 B heapSize=800 B 2023-09-06T20:34:33,482 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/c2f30705e9284eb6809c1c1b52f2a423 is 218, key is session:backup_1694032469137/session:context/1694032473290/Put/seqid=0 2023-09-06T20:34:33,487 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=214 B at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/c2f30705e9284eb6809c1c1b52f2a423 2023-09-06T20:34:33,494 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/c2f30705e9284eb6809c1c1b52f2a423 as hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/c2f30705e9284eb6809c1c1b52f2a423 2023-09-06T20:34:33,500 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/c2f30705e9284eb6809c1c1b52f2a423, entries=1, sequenceid=24, filesize=5.2 K 2023-09-06T20:34:33,501 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(3022): Finished flush of dataSize ~214 B/214, heapSize ~528 B/528, currentSize=0 B/0 for cc8a9b52139566dcd11c25f6d6fdfaad in 24ms, sequenceid=24, compaction requested=true 2023-09-06T20:34:33,512 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/recovered.edits/27.seqid, newMaxSeqId=27, maxSeqId=1 2023-09-06T20:34:33,512 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:34:33,513 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1909): Closed backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:34:33,513 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1622): Region close journal for cc8a9b52139566dcd11c25f6d6fdfaad: 2023-09-06T20:34:33,515 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] handler.UnassignRegionHandler(163): Closed cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:33,515 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=cc8a9b52139566dcd11c25f6d6fdfaad, regionState=CLOSED 2023-09-06T20:34:33,518 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=47, resume processing ppid=46 2023-09-06T20:34:33,518 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=47, ppid=46, state=SUCCESS; CloseRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad, server=jenkins-hbase3.apache.org,44489,1694032433466 in 192 msec 2023-09-06T20:34:33,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=46, resume processing ppid=45 2023-09-06T20:34:33,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=46, ppid=45, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, UNASSIGN in 197 msec 2023-09-06T20:34:33,522 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032473521"}]},"ts":"1694032473521"} 2023-09-06T20:34:33,523 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-09-06T20:34:33,525 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-09-06T20:34:33,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=45, state=SUCCESS; DisableTableProcedure table=backup:system in 219 msec 2023-09-06T20:34:33,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=45 2023-09-06T20:34:33,630 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1694032473629 table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:34:33,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032473630 (current time:1694032473630). 2023-09-06T20:34:33,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:34:33,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1694032473629 VERSION not specified, setting to 2 2023-09-06T20:34:33,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3714ceee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:33,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:33,639 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46830, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:33,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:33,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:34:33,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=48, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032473629 table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:34:33,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032473629 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 48 2023-09-06T20:34:33,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-06T20:34:33,643 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032473629 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:34:33,643 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032473629 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:34:33,645 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032473629 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:34:33,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-06T20:34:33,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-06T20:34:34,052 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032473629 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-09-06T20:34:34,054 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-09-06T20:34:34,054 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-09-06T20:34:34,054 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:34:34,055 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/89e49a43114a459a838964cbd810a62a] hfiles 2023-09-06T20:34:34,055 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/1): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/89e49a43114a459a838964cbd810a62a 2023-09-06T20:34:34,055 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:34:34,057 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/ae2e5dd8863b45f491fb822de06cae0f, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/c2f30705e9284eb6809c1c1b52f2a423, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/f4a9d1ee08d64003a14895000c52e7db] hfiles 2023-09-06T20:34:34,058 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/3): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/ae2e5dd8863b45f491fb822de06cae0f 2023-09-06T20:34:34,058 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/3): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/c2f30705e9284eb6809c1c1b52f2a423 2023-09-06T20:34:34,058 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/3): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/f4a9d1ee08d64003a14895000c52e7db 2023-09-06T20:34:34,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-06T20:34:34,464 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => cc8a9b52139566dcd11c25f6d6fdfaad, NAME => 'backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-09-06T20:34:34,465 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032473629 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:34:34,466 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032473629 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:34:34,466 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1694032473629 2023-09-06T20:34:34,467 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1694032473629 2023-09-06T20:34:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-06T20:34:34,878 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032473629 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:34:34,886 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032473629 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:34:34,886 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1694032473629 to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1694032473629 2023-09-06T20:34:34,888 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032473629 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:34:34,888 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032473629 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 48 2023-09-06T20:34:34,889 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=48, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032473629 table=backup:system type=FLUSH ttl=0 } in 1.2480 sec 2023-09-06T20:34:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-06T20:34:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=49, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032469164 type: FLUSH version: 2 ttl: 0 ) 2023-09-06T20:34:35,778 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(1015): Restore snapshot=snapshot_backup_system as table=backup:system 2023-09-06T20:34:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=49 2023-09-06T20:34:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=49 2023-09-06T20:34:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=49 2023-09-06T20:34:36,195 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/.tabledesc/.tableinfo.0000000001.635 2023-09-06T20:34:36,195 INFO [PEWorker-5 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2023-09-06T20:34:36,196 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:34:36,203 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032469164 type: FLUSH version: 2 ttl: 0 2023-09-06T20:34:36,204 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system 2023-09-06T20:34:36,207 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-09-06T20:34:36,207 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(227): region to restore: cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:36,213 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:34:36,223 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/c2f30705e9284eb6809c1c1b52f2a423 to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/archive/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/c2f30705e9284eb6809c1c1b52f2a423 2023-09-06T20:34:36,224 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:34:36,224 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032469164 type: FLUSH version: 2 ttl: 0 2023-09-06T20:34:36,226 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1694032476226"}]},"ts":"9223372036854775807"} 2023-09-06T20:34:36,229 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-06T20:34:36,229 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => cc8a9b52139566dcd11c25f6d6fdfaad, NAME => 'backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:34:36,229 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1694032476227"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032476227"}]},"ts":"1694032476227"} 2023-09-06T20:34:36,230 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:34:36,231 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-09-06T20:34:36,231 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => cc8a9b52139566dcd11c25f6d6fdfaad, NAME => 'backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:34:36,231 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-09-06T20:34:36,233 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=49, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032469164 type: FLUSH version: 2 ttl: 0 ) in 466 msec 2023-09-06T20:34:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=49 2023-09-06T20:34:36,387 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1694032473629" 2023-09-06T20:34:36,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(378): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1694032473629 2023-09-06T20:34:36,398 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.HMaster$11(2631): Client=jenkins//172.31.12.81 enable backup:system 2023-09-06T20:34:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=50, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-09-06T20:34:36,411 INFO [PEWorker-4 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-09-06T20:34:36,411 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032476411"}]},"ts":"1694032476411"} 2023-09-06T20:34:36,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-09-06T20:34:36,413 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-09-06T20:34:36,416 DEBUG [PEWorker-4 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-09-06T20:34:36,416 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, ASSIGN}] 2023-09-06T20:34:36,417 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, ASSIGN 2023-09-06T20:34:36,418 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=51, ppid=50, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-09-06T20:34:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-09-06T20:34:36,570 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=51 updating hbase:meta row=cc8a9b52139566dcd11c25f6d6fdfaad, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:36,571 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE; OpenRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad, server=jenkins-hbase3.apache.org,44489,1694032433466}] 2023-09-06T20:34:36,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-09-06T20:34:36,727 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(130): Open backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:34:36,727 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7254): Opening region: {ENCODED => cc8a9b52139566dcd11c25f6d6fdfaad, NAME => 'backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:34:36,727 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:34:36,728 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:36,728 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(888): Instantiated backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:34:36,728 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7296): checking encryption for cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:36,728 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7299): checking classloading for cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:36,729 INFO [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:36,730 INFO [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc8a9b52139566dcd11c25f6d6fdfaad columnFamilyName meta 2023-09-06T20:34:36,730 DEBUG [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:34:36,737 DEBUG [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/89e49a43114a459a838964cbd810a62a 2023-09-06T20:34:36,738 INFO [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.HStore(324): Store=cc8a9b52139566dcd11c25f6d6fdfaad/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:34:36,738 INFO [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:36,739 INFO [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc8a9b52139566dcd11c25f6d6fdfaad columnFamilyName session 2023-09-06T20:34:36,739 DEBUG [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:34:36,745 DEBUG [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/ae2e5dd8863b45f491fb822de06cae0f 2023-09-06T20:34:36,750 DEBUG [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/f4a9d1ee08d64003a14895000c52e7db 2023-09-06T20:34:36,750 INFO [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.HStore(324): Store=cc8a9b52139566dcd11c25f6d6fdfaad/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:34:36,751 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:36,752 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:36,753 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-09-06T20:34:36,755 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1076): writing seq id for cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:36,755 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): Opened cc8a9b52139566dcd11c25f6d6fdfaad; next sequenceid=28; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11503420480, jitterRate=0.07133951783180237}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-09-06T20:34:36,756 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(992): Region open journal for cc8a9b52139566dcd11c25f6d6fdfaad: 2023-09-06T20:34:36,757 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad., pid=52, masterSystemTime=1694032476723 2023-09-06T20:34:36,758 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:34:36,758 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(158): Opened backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:34:36,759 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=51 updating hbase:meta row=cc8a9b52139566dcd11c25f6d6fdfaad, regionState=OPEN, openSeqNum=28, regionLocation=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:36,762 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=52, resume processing ppid=51 2023-09-06T20:34:36,762 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=52, ppid=51, state=SUCCESS; OpenRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad, server=jenkins-hbase3.apache.org,44489,1694032433466 in 189 msec 2023-09-06T20:34:36,763 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=51, resume processing ppid=50 2023-09-06T20:34:36,763 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=51, ppid=50, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, ASSIGN in 346 msec 2023-09-06T20:34:36,764 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032476764"}]},"ts":"1694032476764"} 2023-09-06T20:34:36,765 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-09-06T20:34:36,766 INFO [PEWorker-1 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-09-06T20:34:36,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=50, state=SUCCESS; EnableTableProcedure table=backup:system in 366 msec 2023-09-06T20:34:37,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-09-06T20:34:37,015 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(1525): Done restoring backup system table 2023-09-06T20:34:37,016 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:34:37,022 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:34:37,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:34:37,025 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:34:37,025 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-09-06T20:34:37,027 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:34:37,034 ERROR [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(246): Backup backup_1694032469137 failed. Backup session finished. Status: FAILURE 2023-09-06T20:34:37,034 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:37,034 ERROR [Listener at jenkins-hbase3.apache.org/45863 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_1 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:168) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_1 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:126) ~[test-classes/:?] ... 41 more 2023-09-06T20:34:37,040 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] backup.TestIncrementalBackupWithFailures(136): Running stage 2 2023-09-06T20:34:37,063 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3273808f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:37,064 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:34:37,066 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41330, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:34:37,070 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:37,071 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46838, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:37,132 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:34:37,138 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:34:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032477138 (current time:1694032477138). 2023-09-06T20:34:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:34:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:34:37,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16acf96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:37,153 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46844, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:37,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:37,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:34:37,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=53, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:34:37,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 53 2023-09-06T20:34:37,158 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=53, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:34:37,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=53 2023-09-06T20:34:37,159 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=53, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:34:37,161 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=53, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:34:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=53 2023-09-06T20:34:37,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=53 2023-09-06T20:34:37,576 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=53, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:34:37,576 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE; SnapshotRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad}] 2023-09-06T20:34:37,578 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=54, ppid=53, state=RUNNABLE; SnapshotRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:37,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44489 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=54 2023-09-06T20:34:37,730 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:34:37,732 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] regionserver.HRegion(2819): Flushing cc8a9b52139566dcd11c25f6d6fdfaad 2/2 column families, dataSize=341 B heapSize=1.05 KB 2023-09-06T20:34:37,749 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/991095229826401eb83b47fa755c059c is 256, key is session:backup_1694032469137/session:context/1694032477026/Put/seqid=0 2023-09-06T20:34:37,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=53 2023-09-06T20:34:37,844 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] procedure2.ProcedureExecutor(1032): Stored pid=55, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE; InitializeStoreFileTrackerProcedure table=test-1694032442288 2023-09-06T20:34:38,154 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=341 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/991095229826401eb83b47fa755c059c 2023-09-06T20:34:38,161 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/991095229826401eb83b47fa755c059c as hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/991095229826401eb83b47fa755c059c 2023-09-06T20:34:38,167 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/991095229826401eb83b47fa755c059c, entries=2, sequenceid=33, filesize=5.2 K 2023-09-06T20:34:38,168 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] regionserver.HRegion(3022): Finished flush of dataSize ~341 B/341, heapSize ~808 B/808, currentSize=0 B/0 for cc8a9b52139566dcd11c25f6d6fdfaad in 437ms, sequenceid=33, compaction requested=true 2023-09-06T20:34:38,168 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] regionserver.HRegion(2520): Flush status journal for cc8a9b52139566dcd11c25f6d6fdfaad: 2023-09-06T20:34:38,168 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. for snapshot_backup_system completed. 2023-09-06T20:34:38,168 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:34:38,169 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:34:38,169 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/89e49a43114a459a838964cbd810a62a] hfiles 2023-09-06T20:34:38,169 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/89e49a43114a459a838964cbd810a62a for snapshot=snapshot_backup_system 2023-09-06T20:34:38,169 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/f4a9d1ee08d64003a14895000c52e7db, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/ae2e5dd8863b45f491fb822de06cae0f, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/991095229826401eb83b47fa755c059c] hfiles 2023-09-06T20:34:38,169 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/f4a9d1ee08d64003a14895000c52e7db for snapshot=snapshot_backup_system 2023-09-06T20:34:38,169 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/ae2e5dd8863b45f491fb822de06cae0f for snapshot=snapshot_backup_system 2023-09-06T20:34:38,169 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/991095229826401eb83b47fa755c059c for snapshot=snapshot_backup_system 2023-09-06T20:34:38,175 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:34:38,175 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=54 2023-09-06T20:34:38,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.HMaster(3995): Remote procedure done, pid=54 2023-09-06T20:34:38,175 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:38,177 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=54, ppid=53, state=RUNNABLE; SnapshotRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:38,179 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=54, resume processing ppid=53 2023-09-06T20:34:38,179 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=54, ppid=53, state=SUCCESS; SnapshotRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad in 602 msec 2023-09-06T20:34:38,179 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=53, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:34:38,180 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=53, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:34:38,180 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=53, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:34:38,180 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:34:38,181 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:34:38,258 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/.tabledesc/.tableinfo.0000000002.598 2023-09-06T20:34:38,258 INFO [PEWorker-5 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/.tabledesc/.tableinfo.0000000003.640 2023-09-06T20:34:38,259 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=55, state=SUCCESS; InitializeStoreFileTrackerProcedure table=test-1694032442288 in 420 msec 2023-09-06T20:34:38,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=53 2023-09-06T20:34:38,594 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=53, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:34:38,602 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=53, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:34:38,603 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:34:38,604 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=53, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:34:38,604 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 53 2023-09-06T20:34:38,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=53, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.4490 sec 2023-09-06T20:34:39,070 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:34:39,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=53 2023-09-06T20:34:39,265 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(115): Backup backup_1694032477102 started at 1694032479265. 2023-09-06T20:34:39,267 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(119): Backup session backup_1694032477102 has been started. 2023-09-06T20:34:39,268 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] backup.TestBackupBase$IncrementalTableBackupClientForTest(128): For incremental backup, current table set is [test-1694032442288, ns2:test-16940324422881] 2023-09-06T20:34:39,271 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager(76): StartCode 1694032437629for backupID backup_1694032477102 2023-09-06T20:34:39,271 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-09-06T20:34:39,272 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-09-06T20:34:39,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-09-06T20:34:39,273 INFO [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-09-06T20:34:39,273 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-09-06T20:34:39,274 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-09-06T20:34:39,274 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-09-06T20:34:39,275 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:34:39,276 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:34:39,276 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:34:39,276 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:39,277 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-09-06T20:34:39,277 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-09-06T20:34:39,278 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-09-06T20:34:39,278 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:34:39,278 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-09-06T20:34:39,278 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-09-06T20:34:39,278 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-09-06T20:34:39,278 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-09-06T20:34:39,279 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-09-06T20:34:39,280 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-09-06T20:34:39,280 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-09-06T20:34:39,280 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,44489,1694032433466' joining acquired barrier for procedure (rolllog) in zk 2023-09-06T20:34:39,282 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-09-06T20:34:39,282 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:39,282 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:34:39,283 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:34:39,283 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-09-06T20:34:39,283 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:34:39,283 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:34:39,284 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:39,284 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:39,284 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:34:39,285 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,44489,1694032433466' joining acquired barrier for procedure 'rolllog' on coordinator 2023-09-06T20:34:39,285 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@b45ebf0[Count = 0] remaining members to acquire global barrier 2023-09-06T20:34:39,285 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-09-06T20:34:39,285 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-09-06T20:34:39,286 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-09-06T20:34:39,286 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-09-06T20:34:39,286 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-09-06T20:34:39,286 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-09-06T20:34:39,286 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-09-06T20:34:39,286 DEBUG [rs(jenkins-hbase3.apache.org,44489,1694032433466)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:39,286 INFO [rs(jenkins-hbase3.apache.org,44489,1694032433466)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1694032457863 highest: 1694032457884 on jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:39,287 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0:(num 1694032457863) roll requested 2023-09-06T20:34:39,299 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032479287, exclude list is [], retry=0 2023-09-06T20:34:39,303 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK] 2023-09-06T20:34:39,305 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032457863 with entries=124, filesize=18.41 KB; new WAL /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032479287 2023-09-06T20:34:39,305 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK]] 2023-09-06T20:34:39,306 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032457863 is not closed yet, will try archiving it next time 2023-09-06T20:34:39,307 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1:(num 1694032457884) roll requested 2023-09-06T20:34:39,307 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032435590 to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/oldWALs/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032435590 2023-09-06T20:34:39,318 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032479307, exclude list is [], retry=0 2023-09-06T20:34:39,321 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK] 2023-09-06T20:34:39,323 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032457884 with entries=7, filesize=1.16 KB; new WAL /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032479307 2023-09-06T20:34:39,323 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK]] 2023-09-06T20:34:39,323 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032457884 is not closed yet, will try archiving it next time 2023-09-06T20:34:39,323 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta:.meta(num 1694032457903) roll requested 2023-09-06T20:34:39,325 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032457884 not finished, retry = 0 2023-09-06T20:34:39,334 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta.1694032479323.meta, exclude list is [], retry=0 2023-09-06T20:34:39,337 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK] 2023-09-06T20:34:39,339 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta.1694032457903.meta with entries=10, filesize=2.50 KB; new WAL /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta.1694032479323.meta 2023-09-06T20:34:39,339 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK]] 2023-09-06T20:34:39,339 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta.1694032457903.meta is not closed yet, will try archiving it next time 2023-09-06T20:34:39,347 DEBUG [rs(jenkins-hbase3.apache.org,44489,1694032433466)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2023-09-06T20:34:39,347 INFO [rs(jenkins-hbase3.apache.org,44489,1694032433466)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1694032479287 on jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:39,355 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-09-06T20:34:39,355 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,44489,1694032433466' in zk 2023-09-06T20:34:39,359 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-09-06T20:34:39,359 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:34:39,359 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:39,359 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-09-06T20:34:39,360 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:34:39,360 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:34:39,361 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:34:39,361 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:34:39,361 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:39,361 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:39,362 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:34:39,362 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:39,362 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:39,363 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,44489,1694032433466': 2023-09-06T20:34:39,363 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,44489,1694032433466' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-09-06T20:34:39,363 INFO [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-09-06T20:34:39,363 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-09-06T20:34:39,363 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-09-06T20:34:39,363 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-09-06T20:34:39,363 INFO [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-09-06T20:34:39,365 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-09-06T20:34:39,365 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-09-06T20:34:39,365 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:34:39,365 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:34:39,365 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:34:39,365 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:34:39,365 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:34:39,365 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:34:39,366 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:34:39,366 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:39,366 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:34:39,367 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:39,367 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:39,367 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:34:39,368 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:39,368 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:39,375 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:34:39,375 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:34:39,375 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:34:39,375 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-09-06T20:34:39,375 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-09-06T20:34:39,376 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:34:39,376 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:34:39,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-09-06T20:34:39,479 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:44489=1694032437629} newestTimestamps: {jenkins-hbase3.apache.org:44489=1694032457884} 2023-09-06T20:34:39,481 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta.1694032435949.meta 2023-09-06T20:34:39,481 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta.1694032457903.meta 2023-09-06T20:34:39,482 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta.1694032479323.meta 2023-09-06T20:34:39,482 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032457863 2023-09-06T20:34:39,482 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032479287 2023-09-06T20:34:39,482 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032457884 2023-09-06T20:34:39,482 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032479307 2023-09-06T20:34:39,483 WARN [Listener at jenkins-hbase3.apache.org/45863 {}] util.BackupUtils(373): Skip log file (can't parse): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/oldWALs/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032435590 2023-09-06T20:34:39,895 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] util.BackupUtils(146): Attempting to copy table info for:ns2:test-16940324422881 target: hdfs://localhost:38135/backupUT/backup_1694032477102/ns2/test-16940324422881 descriptor: 'ns2:test-16940324422881', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:34:39,895 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-09-06T20:34:39,897 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] util.BackupUtils(151): Starting to write region info for table ns2:test-16940324422881 2023-09-06T20:34:40,303 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] util.BackupUtils(158): Finished writing region info for table ns2:test-16940324422881 2023-09-06T20:34:40,313 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] util.BackupUtils(146): Attempting to copy table info for:test-1694032442288 target: hdfs://localhost:38135/backupUT/backup_1694032477102/default/test-1694032442288 descriptor: 'test-1694032442288', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:34:40,313 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-09-06T20:34:40,315 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] util.BackupUtils(151): Starting to write region info for table test-1694032442288 2023-09-06T20:34:40,320 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] util.BackupUtils(158): Finished writing region info for table test-1694032442288 2023-09-06T20:34:40,328 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:38135/backupUT/.tmp/backup_1694032477102 from hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032457863;hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032457884 2023-09-06T20:34:40,338 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70eb13bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:40,340 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:34:40,342 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:43120, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:34:40,356 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-09-06T20:34:40,356 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1694032442288 2023-09-06T20:34:40,361 DEBUG [hconnection-0x5e1b0af5-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:40,362 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:55646, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:40,365 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1694032442288: test-1694032442288; 2023-09-06T20:34:40,365 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16940324422881 2023-09-06T20:34:40,367 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16940324422881: ns2:test-16940324422881; 2023-09-06T20:34:40,367 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.HFileOutputFormat2(694): Configuring 2 reduce partitions to match current region count for all tables 2023-09-06T20:34:40,368 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/hbase-staging/partitions_59473ca1-5a89-46d4-8e76-96acda5c1847 2023-09-06T20:34:40,807 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:40,807 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:40,808 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:40,809 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:41,808 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/hadoop-8974922225868357732.jar 2023-09-06T20:34:41,809 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:41,809 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:41,810 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:41,810 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:41,811 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:41,811 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:41,812 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:41,812 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:41,812 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:41,813 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:34:41,814 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:34:41,814 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:34:41,815 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:34:41,815 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:34:41,816 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:34:41,816 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:34:41,816 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:34:41,817 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:34:41,817 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:34:41,818 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:34:41,818 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:34:41,820 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:41,821 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:41,821 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:41,822 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:41,822 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:41,822 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:41,823 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:34:41,823 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: default:test-1694032442288,ns2:test-16940324422881 2023-09-06T20:34:41,823 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:41,824 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:34:42,786 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/hadoop-4935480113943729782.jar 2023-09-06T20:34:42,807 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032457863 for WAL files 2023-09-06T20:34:42,808 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032457863 2023-09-06T20:34:42,808 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032457884 for WAL files 2023-09-06T20:34:42,809 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032457884 2023-09-06T20:34:42,891 WARN [Listener at jenkins-hbase3.apache.org/45863 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/mapreduce.cluster.local.dir/1694032482857/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:34:42,891 WARN [Listener at jenkins-hbase3.apache.org/45863 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/mapreduce.cluster.local.dir/1694032482857/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:34:43,007 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032457863 for hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032457863 (-9223372036854775808:9223372036854775807) length:18860 2023-09-06T20:34:43,022 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-09-06T20:34:43,023 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-09-06T20:34:43,064 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032457884 for hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032457884 (-9223372036854775808:9223372036854775807) length:1198 2023-09-06T20:34:43,067 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-09-06T20:34:43,068 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-09-06T20:34:43,172 DEBUG [pool-225-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6208398, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:43,173 DEBUG [hconnection-0x14e16c87-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:43,176 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:55656, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:43,178 DEBUG [pool-225-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:43,179 DEBUG [pool-225-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-t20] 2023-09-06T20:34:43,179 DEBUG [pool-225-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-09-06T20:34:43,186 DEBUG [pool-225-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/backupUT/.tmp/backup_1694032477102/_temporary/0/_temporary/attempt_local650440684_0003_r_000000_0/ns2/test-16940324422881/f/b54cc8f4d67f45d5bffecb434e726dbc is 38, key is row-t20/f:q1/1694032468944/Put/seqid=0 2023-09-06T20:34:43,188 WARN [Thread-912 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:44489] Specified favored nodes: [jenkins-hbase3.apache.org:44489] 2023-09-06T20:34:43,622 DEBUG [pool-225-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f73071e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:43,624 DEBUG [hconnection-0x172b4d1a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:43,625 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:55666, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:43,627 DEBUG [pool-225-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:43,628 DEBUG [pool-225-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-test-1694032442288-1-0] 2023-09-06T20:34:43,628 DEBUG [pool-225-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-09-06T20:34:43,637 DEBUG [pool-225-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/backupUT/.tmp/backup_1694032477102/_temporary/0/_temporary/attempt_local650440684_0003_r_000001_0/default/test-1694032442288/f/22eff241f4a64facb77a0c93882924ef is 59, key is row-test-1694032442288-1-10/f:q1/1694032468833/Put/seqid=0 2023-09-06T20:34:43,639 WARN [Thread-922 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:44489] Specified favored nodes: [jenkins-hbase3.apache.org:44489] 2023-09-06T20:34:44,929 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:38135/backupUT 2023-09-06T20:34:44,929 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1694032477102 2023-09-06T20:34:44,930 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-09-06T20:34:44,944 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:38135/backupUT/.tmp/backup_1694032477102, hdfs://localhost:38135/backupUT] 2023-09-06T20:34:46,088 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-09-06T20:34:46,090 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1694032477102 2023-09-06T20:34:46,090 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 16003 bytes copied." 2023-09-06T20:34:46,090 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local1603134290_0004 completed: true true 2023-09-06T20:34:46,096 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=3790848 FILE: Number of bytes written=5084322 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=189984 HDFS: Number of bytes written=2364105 HDFS: Number of read operations=690 HDFS: Number of large read operations=0 HDFS: Number of write operations=246 Map-Reduce Framework Map input records=10 Map output records=0 Input split bytes=342 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1532493824 File Input Format Counters Bytes Read=2444 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=16003 Bytes Expected=16003 Files Copied=3 DIR_COPY=7 2023-09-06T20:34:46,096 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:38135/backupUT/.tmp/backup_1694032477102 to hdfs://localhost:38135/backupUT finished. 2023-09-06T20:34:46,097 ERROR [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(228): Unexpected Exception : Failed stage stage_2 in testing java.io.IOException: Failed stage stage_2 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:34:46,097 ERROR [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(240): BackupId=backup_1694032477102,startts=1694032479265,failedts=1694032486097,failedphase=INCREMENTAL_COPY,failedmessage=Failed stage stage_2 in testing 2023-09-06T20:34:46,098 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-09-06T20:34:46,103 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable backup:system 2023-09-06T20:34:46,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-09-06T20:34:46,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=56 2023-09-06T20:34:46,107 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032486107"}]},"ts":"1694032486107"} 2023-09-06T20:34:46,108 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-09-06T20:34:46,110 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-09-06T20:34:46,110 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, UNASSIGN}] 2023-09-06T20:34:46,111 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=57, ppid=56, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, UNASSIGN 2023-09-06T20:34:46,112 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=57 updating hbase:meta row=cc8a9b52139566dcd11c25f6d6fdfaad, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:46,113 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE; CloseRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad, server=jenkins-hbase3.apache.org,44489,1694032433466}] 2023-09-06T20:34:46,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=56 2023-09-06T20:34:46,265 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(119): Close cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:46,265 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1668): Closing cc8a9b52139566dcd11c25f6d6fdfaad, disabling compactions & flushes 2023-09-06T20:34:46,265 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1690): Closing region backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:34:46,265 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:34:46,265 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. after waiting 0 ms 2023-09-06T20:34:46,265 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:34:46,265 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(2819): Flushing cc8a9b52139566dcd11c25f6d6fdfaad 2/2 column families, dataSize=540 B heapSize=1.24 KB 2023-09-06T20:34:46,285 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/meta/85a4183d70c3461999a1a6acef218fb9 is 116, key is rslogts:hdfs://localhost:38135/backupUT\x00jenkins-hbase3.apache.org:44489/meta:rs-log-ts/1694032479355/Put/seqid=0 2023-09-06T20:34:46,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=56 2023-09-06T20:34:46,690 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=112 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/meta/85a4183d70c3461999a1a6acef218fb9 2023-09-06T20:34:46,699 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/f41ee2e8106b419e8bf0a94b76a69510 is 218, key is session:backup_1694032477102/session:context/1694032486089/Put/seqid=0 2023-09-06T20:34:46,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=56 2023-09-06T20:34:47,104 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=428 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/f41ee2e8106b419e8bf0a94b76a69510 2023-09-06T20:34:47,111 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/meta/85a4183d70c3461999a1a6acef218fb9 as hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/85a4183d70c3461999a1a6acef218fb9 2023-09-06T20:34:47,118 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/85a4183d70c3461999a1a6acef218fb9, entries=1, sequenceid=39, filesize=5.3 K 2023-09-06T20:34:47,119 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/f41ee2e8106b419e8bf0a94b76a69510 as hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/f41ee2e8106b419e8bf0a94b76a69510 2023-09-06T20:34:47,126 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/f41ee2e8106b419e8bf0a94b76a69510, entries=1, sequenceid=39, filesize=5.2 K 2023-09-06T20:34:47,127 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(3022): Finished flush of dataSize ~540 B/540, heapSize ~1.21 KB/1240, currentSize=0 B/0 for cc8a9b52139566dcd11c25f6d6fdfaad in 862ms, sequenceid=39, compaction requested=true 2023-09-06T20:34:47,140 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/recovered.edits/42.seqid, newMaxSeqId=42, maxSeqId=27 2023-09-06T20:34:47,140 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:34:47,142 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1909): Closed backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:34:47,142 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1622): Region close journal for cc8a9b52139566dcd11c25f6d6fdfaad: 2023-09-06T20:34:47,144 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(163): Closed cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:47,145 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=57 updating hbase:meta row=cc8a9b52139566dcd11c25f6d6fdfaad, regionState=CLOSED 2023-09-06T20:34:47,147 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=58, resume processing ppid=57 2023-09-06T20:34:47,147 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=58, ppid=57, state=SUCCESS; CloseRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad, server=jenkins-hbase3.apache.org,44489,1694032433466 in 1.0330 sec 2023-09-06T20:34:47,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=57, resume processing ppid=56 2023-09-06T20:34:47,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=57, ppid=56, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, UNASSIGN in 1.0370 sec 2023-09-06T20:34:47,150 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032487149"}]},"ts":"1694032487149"} 2023-09-06T20:34:47,151 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-09-06T20:34:47,153 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-09-06T20:34:47,154 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=56, state=SUCCESS; DisableTableProcedure table=backup:system in 1.0500 sec 2023-09-06T20:34:47,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=56 2023-09-06T20:34:47,220 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1694032487220 table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:34:47,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032487220 (current time:1694032487220). 2023-09-06T20:34:47,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:34:47,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1694032487220 VERSION not specified, setting to 2 2023-09-06T20:34:47,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ed4a459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:47,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:47,231 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:55678, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:47,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:47,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:34:47,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=59, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032487220 table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:34:47,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032487220 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 59 2023-09-06T20:34:47,235 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032487220 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:34:47,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=59 2023-09-06T20:34:47,236 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032487220 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:34:47,238 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032487220 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:34:47,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=59 2023-09-06T20:34:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=59 2023-09-06T20:34:47,651 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032487220 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-09-06T20:34:47,652 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-09-06T20:34:47,652 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-09-06T20:34:47,652 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:34:47,653 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/85a4183d70c3461999a1a6acef218fb9, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/89e49a43114a459a838964cbd810a62a] hfiles 2023-09-06T20:34:47,653 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/2): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/85a4183d70c3461999a1a6acef218fb9 2023-09-06T20:34:47,653 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/2): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/89e49a43114a459a838964cbd810a62a 2023-09-06T20:34:47,654 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:34:47,656 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/991095229826401eb83b47fa755c059c, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/ae2e5dd8863b45f491fb822de06cae0f, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/f41ee2e8106b419e8bf0a94b76a69510, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/f4a9d1ee08d64003a14895000c52e7db] hfiles 2023-09-06T20:34:47,656 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/4): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/991095229826401eb83b47fa755c059c 2023-09-06T20:34:47,656 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/4): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/ae2e5dd8863b45f491fb822de06cae0f 2023-09-06T20:34:47,656 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/4): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/f41ee2e8106b419e8bf0a94b76a69510 2023-09-06T20:34:47,656 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (4/4): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/f4a9d1ee08d64003a14895000c52e7db 2023-09-06T20:34:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=59 2023-09-06T20:34:48,062 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => cc8a9b52139566dcd11c25f6d6fdfaad, NAME => 'backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-09-06T20:34:48,063 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032487220 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:34:48,063 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032487220 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:34:48,063 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1694032487220 2023-09-06T20:34:48,064 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1694032487220 2023-09-06T20:34:48,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=59 2023-09-06T20:34:48,475 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032487220 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:34:48,483 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032487220 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:34:48,484 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1694032487220 to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1694032487220 2023-09-06T20:34:48,485 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032487220 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:34:48,485 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032487220 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 59 2023-09-06T20:34:48,486 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=59, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032487220 table=backup:system type=FLUSH ttl=0 } in 1.2520 sec 2023-09-06T20:34:48,488 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-09-06T20:34:48,489 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-09-06T20:34:49,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=59 2023-09-06T20:34:49,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=60, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032477138 type: FLUSH version: 2 ttl: 0 ) 2023-09-06T20:34:49,354 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(1015): Restore snapshot=snapshot_backup_system as table=backup:system 2023-09-06T20:34:49,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-09-06T20:34:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-09-06T20:34:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-09-06T20:34:49,769 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2023-09-06T20:34:49,770 INFO [PEWorker-2 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/.tabledesc/.tableinfo.0000000003.635 2023-09-06T20:34:49,771 INFO [PEWorker-2 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:34:49,773 INFO [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032477138 type: FLUSH version: 2 ttl: 0 2023-09-06T20:34:49,773 DEBUG [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system 2023-09-06T20:34:49,777 DEBUG [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-09-06T20:34:49,777 INFO [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(227): region to restore: cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:49,784 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/85a4183d70c3461999a1a6acef218fb9 to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/archive/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/85a4183d70c3461999a1a6acef218fb9 2023-09-06T20:34:49,784 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:34:49,788 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/f41ee2e8106b419e8bf0a94b76a69510 to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/archive/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/f41ee2e8106b419e8bf0a94b76a69510 2023-09-06T20:34:49,788 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:34:49,788 INFO [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032477138 type: FLUSH version: 2 ttl: 0 2023-09-06T20:34:49,789 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1694032489789"}]},"ts":"9223372036854775807"} 2023-09-06T20:34:49,791 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-06T20:34:49,791 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => cc8a9b52139566dcd11c25f6d6fdfaad, NAME => 'backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:34:49,792 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1694032489790"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032489790"}]},"ts":"1694032489790"} 2023-09-06T20:34:49,793 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:34:49,793 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-09-06T20:34:49,793 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => cc8a9b52139566dcd11c25f6d6fdfaad, NAME => 'backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:34:49,793 INFO [PEWorker-2 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-09-06T20:34:49,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=60, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032477138 type: FLUSH version: 2 ttl: 0 ) in 442 msec 2023-09-06T20:34:49,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-09-06T20:34:49,962 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1694032487220" 2023-09-06T20:34:49,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(378): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1694032487220 2023-09-06T20:34:49,966 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.HMaster$11(2631): Client=jenkins//172.31.12.81 enable backup:system 2023-09-06T20:34:49,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=61, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-09-06T20:34:49,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=61 2023-09-06T20:34:49,970 INFO [PEWorker-5 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-09-06T20:34:49,971 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032489971"}]},"ts":"1694032489971"} 2023-09-06T20:34:49,972 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-09-06T20:34:49,975 DEBUG [PEWorker-5 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-09-06T20:34:49,975 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, ASSIGN}] 2023-09-06T20:34:49,977 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, ASSIGN 2023-09-06T20:34:49,978 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-09-06T20:34:50,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=61 2023-09-06T20:34:50,129 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=62 updating hbase:meta row=cc8a9b52139566dcd11c25f6d6fdfaad, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:50,131 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE; OpenRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad, server=jenkins-hbase3.apache.org,44489,1694032433466}] 2023-09-06T20:34:50,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=61 2023-09-06T20:34:50,286 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(130): Open backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:34:50,286 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7254): Opening region: {ENCODED => cc8a9b52139566dcd11c25f6d6fdfaad, NAME => 'backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:34:50,287 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:34:50,287 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:50,287 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(888): Instantiated backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:34:50,287 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7296): checking encryption for cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:50,287 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7299): checking classloading for cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:50,288 INFO [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:50,289 INFO [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc8a9b52139566dcd11c25f6d6fdfaad columnFamilyName meta 2023-09-06T20:34:50,289 DEBUG [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:34:50,295 DEBUG [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/89e49a43114a459a838964cbd810a62a 2023-09-06T20:34:50,295 INFO [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.HStore(324): Store=cc8a9b52139566dcd11c25f6d6fdfaad/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:34:50,295 INFO [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:50,296 INFO [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc8a9b52139566dcd11c25f6d6fdfaad columnFamilyName session 2023-09-06T20:34:50,296 DEBUG [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:34:50,303 DEBUG [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/991095229826401eb83b47fa755c059c 2023-09-06T20:34:50,307 DEBUG [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/ae2e5dd8863b45f491fb822de06cae0f 2023-09-06T20:34:50,312 DEBUG [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/f4a9d1ee08d64003a14895000c52e7db 2023-09-06T20:34:50,312 INFO [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.HStore(324): Store=cc8a9b52139566dcd11c25f6d6fdfaad/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:34:50,313 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:50,314 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:50,315 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-09-06T20:34:50,316 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1076): writing seq id for cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:50,317 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): Opened cc8a9b52139566dcd11c25f6d6fdfaad; next sequenceid=43; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=12024380640, jitterRate=0.11985771358013153}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-09-06T20:34:50,318 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(992): Region open journal for cc8a9b52139566dcd11c25f6d6fdfaad: 2023-09-06T20:34:50,318 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad., pid=63, masterSystemTime=1694032490283 2023-09-06T20:34:50,321 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.CompactSplit(403): Add compact mark for store cc8a9b52139566dcd11c25f6d6fdfaad:session, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:34:50,322 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:34:50,322 DEBUG [RS:0;jenkins-hbase3:44489-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:34:50,323 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:34:50,324 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(158): Opened backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:34:50,324 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=62 updating hbase:meta row=cc8a9b52139566dcd11c25f6d6fdfaad, regionState=OPEN, openSeqNum=43, regionLocation=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:50,325 DEBUG [RS:0;jenkins-hbase3:44489-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 16169 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:34:50,327 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=63, resume processing ppid=62 2023-09-06T20:34:50,327 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=63, ppid=62, state=SUCCESS; OpenRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad, server=jenkins-hbase3.apache.org,44489,1694032433466 in 194 msec 2023-09-06T20:34:50,327 DEBUG [RS:0;jenkins-hbase3:44489-shortCompactions-0 {}] regionserver.HStore(1534): cc8a9b52139566dcd11c25f6d6fdfaad/session is initiating minor compaction (all files) 2023-09-06T20:34:50,328 INFO [RS:0;jenkins-hbase3:44489-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of cc8a9b52139566dcd11c25f6d6fdfaad/session in backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:34:50,328 INFO [RS:0;jenkins-hbase3:44489-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/f4a9d1ee08d64003a14895000c52e7db, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/ae2e5dd8863b45f491fb822de06cae0f, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/991095229826401eb83b47fa755c059c] into tmpdir=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp, totalSize=15.8 K 2023-09-06T20:34:50,328 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=62, resume processing ppid=61 2023-09-06T20:34:50,328 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=62, ppid=61, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, ASSIGN in 352 msec 2023-09-06T20:34:50,329 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032490329"}]},"ts":"1694032490329"} 2023-09-06T20:34:50,329 DEBUG [RS:0;jenkins-hbase3:44489-shortCompactions-0 {}] compactions.Compactor(223): Compacting f4a9d1ee08d64003a14895000c52e7db, keycount=1, bloomtype=ROW, size=4.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1694032455632 2023-09-06T20:34:50,329 DEBUG [RS:0;jenkins-hbase3:44489-shortCompactions-0 {}] compactions.Compactor(223): Compacting ae2e5dd8863b45f491fb822de06cae0f, keycount=3, bloomtype=ROW, size=5.7 K, encoding=NONE, compression=NONE, seqNum=20, earliestPutTs=1694032468780 2023-09-06T20:34:50,330 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-09-06T20:34:50,330 DEBUG [RS:0;jenkins-hbase3:44489-shortCompactions-0 {}] compactions.Compactor(223): Compacting 991095229826401eb83b47fa755c059c, keycount=2, bloomtype=ROW, size=5.2 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1694032477026 2023-09-06T20:34:50,332 INFO [PEWorker-1 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-09-06T20:34:50,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=61, state=SUCCESS; EnableTableProcedure table=backup:system in 366 msec 2023-09-06T20:34:50,351 INFO [RS:0;jenkins-hbase3:44489-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc8a9b52139566dcd11c25f6d6fdfaad#session#compaction#10 average throughput is 0.95 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:34:50,352 DEBUG [RS:0;jenkins-hbase3:44489-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/22fa315304564a1397a14e4ba5bc21c8 is 459, key is session:backup_1694032451196/session:context/1694032468780/Put/seqid=0 2023-09-06T20:34:50,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=61 2023-09-06T20:34:50,573 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(1525): Done restoring backup system table 2023-09-06T20:34:50,573 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:34:50,578 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:34:50,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:34:50,581 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:34:50,582 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: INCREMENTAL_COPY 2023-09-06T20:34:50,582 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:38135/backupUT/backup_1694032477102/ns2/test-16940324422881 done. 2023-09-06T20:34:50,583 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(208): hdfs://localhost:38135/backupUT/backup_1694032477102/ns2 is empty, remove it. 2023-09-06T20:34:50,584 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:38135/backupUT/backup_1694032477102/default/test-1694032442288 done. 2023-09-06T20:34:50,585 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(208): hdfs://localhost:38135/backupUT/backup_1694032477102/default is empty, remove it. 2023-09-06T20:34:50,586 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:34:50,592 ERROR [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(246): Backup backup_1694032477102 failed. Backup session finished. Status: FAILURE 2023-09-06T20:34:50,592 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:50,593 ERROR [Listener at jenkins-hbase3.apache.org/45863 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_2 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:168) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_2 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:137) ~[test-classes/:?] ... 41 more 2023-09-06T20:34:50,597 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] backup.TestIncrementalBackupWithFailures(136): Running stage 3 2023-09-06T20:34:50,618 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56c150c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:50,620 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:34:50,622 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42036, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:34:50,625 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:50,627 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41542, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:50,671 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:34:50,675 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:34:50,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032490675 (current time:1694032490675). 2023-09-06T20:34:50,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:34:50,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:34:50,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af9ef74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:50,684 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41548, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:50,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:50,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:34:50,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=64, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:34:50,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-09-06T20:34:50,687 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:34:50,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-09-06T20:34:50,688 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:34:50,690 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:34:50,763 DEBUG [RS:0;jenkins-hbase3:44489-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/22fa315304564a1397a14e4ba5bc21c8 as hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/22fa315304564a1397a14e4ba5bc21c8 2023-09-06T20:34:50,778 DEBUG [RS:0;jenkins-hbase3:44489-shortCompactions-0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-09-06T20:34:50,779 INFO [RS:0;jenkins-hbase3:44489-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in cc8a9b52139566dcd11c25f6d6fdfaad/session of cc8a9b52139566dcd11c25f6d6fdfaad into 22fa315304564a1397a14e4ba5bc21c8(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:34:50,779 DEBUG [RS:0;jenkins-hbase3:44489-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for cc8a9b52139566dcd11c25f6d6fdfaad: 2023-09-06T20:34:50,779 INFO [RS:0;jenkins-hbase3:44489-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad., storeName=cc8a9b52139566dcd11c25f6d6fdfaad/session, priority=13, startTime=1694032490320; duration=0sec 2023-09-06T20:34:50,780 DEBUG [RS:0;jenkins-hbase3:44489-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:34:50,780 DEBUG [RS:0;jenkins-hbase3:44489-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc8a9b52139566dcd11c25f6d6fdfaad:session 2023-09-06T20:34:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-09-06T20:34:50,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-09-06T20:34:51,097 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:34:51,098 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad}] 2023-09-06T20:34:51,100 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:51,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44489 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=65 2023-09-06T20:34:51,251 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:34:51,252 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2819): Flushing cc8a9b52139566dcd11c25f6d6fdfaad 2/2 column families, dataSize=341 B heapSize=1.05 KB 2023-09-06T20:34:51,256 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/7772e4890a534e54a647e460587efe32 is 256, key is session:backup_1694032477102/session:context/1694032490586/Put/seqid=0 2023-09-06T20:34:51,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-09-06T20:34:51,661 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=341 B at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/7772e4890a534e54a647e460587efe32 2023-09-06T20:34:51,668 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/7772e4890a534e54a647e460587efe32 as hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/7772e4890a534e54a647e460587efe32 2023-09-06T20:34:51,673 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/7772e4890a534e54a647e460587efe32, entries=2, sequenceid=49, filesize=5.2 K 2023-09-06T20:34:51,674 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(3022): Finished flush of dataSize ~341 B/341, heapSize ~808 B/808, currentSize=0 B/0 for cc8a9b52139566dcd11c25f6d6fdfaad in 423ms, sequenceid=49, compaction requested=false 2023-09-06T20:34:51,676 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2520): Flush status journal for cc8a9b52139566dcd11c25f6d6fdfaad: 2023-09-06T20:34:51,676 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. for snapshot_backup_system completed. 2023-09-06T20:34:51,676 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:34:51,676 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:34:51,676 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/89e49a43114a459a838964cbd810a62a] hfiles 2023-09-06T20:34:51,676 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/89e49a43114a459a838964cbd810a62a for snapshot=snapshot_backup_system 2023-09-06T20:34:51,676 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/22fa315304564a1397a14e4ba5bc21c8, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/7772e4890a534e54a647e460587efe32] hfiles 2023-09-06T20:34:51,676 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/22fa315304564a1397a14e4ba5bc21c8 for snapshot=snapshot_backup_system 2023-09-06T20:34:51,676 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/7772e4890a534e54a647e460587efe32 for snapshot=snapshot_backup_system 2023-09-06T20:34:51,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-09-06T20:34:52,085 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:34:52,085 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=65 2023-09-06T20:34:52,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.HMaster(3995): Remote procedure done, pid=65 2023-09-06T20:34:52,086 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:52,087 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:34:52,089 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=65, resume processing ppid=64 2023-09-06T20:34:52,089 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:34:52,089 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=65, ppid=64, state=SUCCESS; SnapshotRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad in 991 msec 2023-09-06T20:34:52,090 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:34:52,090 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:34:52,090 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:34:52,091 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:34:52,507 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:34:52,514 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:34:52,514 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:34:52,516 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:34:52,516 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-09-06T20:34:52,517 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=64, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8300 sec 2023-09-06T20:34:52,684 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:34:52,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-09-06T20:34:52,794 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(115): Backup backup_1694032490651 started at 1694032492793. 2023-09-06T20:34:52,795 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(119): Backup session backup_1694032490651 has been started. 2023-09-06T20:34:52,797 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] backup.TestBackupBase$IncrementalTableBackupClientForTest(128): For incremental backup, current table set is [test-1694032442288, ns2:test-16940324422881] 2023-09-06T20:34:52,799 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager(76): StartCode 1694032437629for backupID backup_1694032490651 2023-09-06T20:34:52,799 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-09-06T20:34:52,800 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-09-06T20:34:52,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-09-06T20:34:52,801 INFO [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-09-06T20:34:52,801 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-09-06T20:34:52,802 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-09-06T20:34:52,802 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-09-06T20:34:52,803 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:34:52,804 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:52,804 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:34:52,804 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:34:52,804 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-09-06T20:34:52,804 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-09-06T20:34:52,805 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-09-06T20:34:52,805 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:34:52,805 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-09-06T20:34:52,805 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-09-06T20:34:52,805 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-09-06T20:34:52,805 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-09-06T20:34:52,805 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-09-06T20:34:52,806 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-09-06T20:34:52,806 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-09-06T20:34:52,806 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,44489,1694032433466' joining acquired barrier for procedure (rolllog) in zk 2023-09-06T20:34:52,810 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-09-06T20:34:52,810 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:52,810 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:34:52,810 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:34:52,810 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-09-06T20:34:52,810 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:34:52,811 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:34:52,811 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:52,811 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:52,811 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:34:52,812 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,44489,1694032433466' joining acquired barrier for procedure 'rolllog' on coordinator 2023-09-06T20:34:52,812 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@2361fabc[Count = 0] remaining members to acquire global barrier 2023-09-06T20:34:52,812 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-09-06T20:34:52,812 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-09-06T20:34:52,813 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-09-06T20:34:52,813 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-09-06T20:34:52,813 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-09-06T20:34:52,813 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-09-06T20:34:52,813 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-09-06T20:34:52,813 DEBUG [rs(jenkins-hbase3.apache.org,44489,1694032433466)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:52,814 INFO [rs(jenkins-hbase3.apache.org,44489,1694032433466)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1694032479287 highest: 1694032479307 on jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:52,814 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0:(num 1694032479287) roll requested 2023-09-06T20:34:52,825 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032492814, exclude list is [], retry=0 2023-09-06T20:34:52,829 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK] 2023-09-06T20:34:52,830 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032479287 with entries=13, filesize=3.56 KB; new WAL /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032492814 2023-09-06T20:34:52,831 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK]] 2023-09-06T20:34:52,831 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032479287 is not closed yet, will try archiving it next time 2023-09-06T20:34:52,831 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1:(num 1694032479307) roll requested 2023-09-06T20:34:52,842 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032492831, exclude list is [], retry=0 2023-09-06T20:34:52,846 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK] 2023-09-06T20:34:52,847 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032479307 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032492831 2023-09-06T20:34:52,847 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK]] 2023-09-06T20:34:52,847 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032479307 is not closed yet, will try archiving it next time 2023-09-06T20:34:52,847 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta:.meta(num 1694032479323) roll requested 2023-09-06T20:34:52,849 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032479307 not finished, retry = 0 2023-09-06T20:34:52,859 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta.1694032492847.meta, exclude list is [], retry=0 2023-09-06T20:34:52,862 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK] 2023-09-06T20:34:52,863 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta.1694032479323.meta with entries=10, filesize=2.50 KB; new WAL /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta.1694032492847.meta 2023-09-06T20:34:52,863 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK]] 2023-09-06T20:34:52,863 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta.1694032479323.meta is not closed yet, will try archiving it next time 2023-09-06T20:34:52,874 DEBUG [rs(jenkins-hbase3.apache.org,44489,1694032433466)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2023-09-06T20:34:52,874 INFO [rs(jenkins-hbase3.apache.org,44489,1694032433466)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1694032492814 on jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:52,882 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-09-06T20:34:52,882 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,44489,1694032433466' in zk 2023-09-06T20:34:52,884 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-09-06T20:34:52,884 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:34:52,884 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:52,884 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-09-06T20:34:52,885 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:34:52,885 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:34:52,885 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:34:52,886 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:34:52,886 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:52,886 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:52,887 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:34:52,887 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:52,887 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:52,888 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,44489,1694032433466': 2023-09-06T20:34:52,888 INFO [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-09-06T20:34:52,888 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,44489,1694032433466' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-09-06T20:34:52,888 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-09-06T20:34:52,888 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-09-06T20:34:52,888 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-09-06T20:34:52,888 INFO [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-09-06T20:34:52,890 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-09-06T20:34:52,890 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-09-06T20:34:52,890 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:34:52,890 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:34:52,890 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:34:52,890 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:34:52,890 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:34:52,890 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:34:52,890 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:34:52,891 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:52,891 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:34:52,891 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:52,892 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:52,892 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:34:52,892 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:34:52,893 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:34:52,898 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:34:52,898 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:34:52,898 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:34:52,898 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-09-06T20:34:52,898 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-09-06T20:34:52,899 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:34:52,899 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:34:52,950 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032479307 to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/oldWALs/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032479307 2023-09-06T20:34:53,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-09-06T20:34:53,002 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:44489=1694032437629} newestTimestamps: {jenkins-hbase3.apache.org:44489=1694032479307} 2023-09-06T20:34:53,004 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta.1694032435949.meta 2023-09-06T20:34:53,004 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta.1694032457903.meta 2023-09-06T20:34:53,004 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta.1694032479323.meta 2023-09-06T20:34:53,004 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta.1694032492847.meta 2023-09-06T20:34:53,004 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032457863 2023-09-06T20:34:53,004 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032479287 2023-09-06T20:34:53,004 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032492814 2023-09-06T20:34:53,004 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032457884 2023-09-06T20:34:53,004 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032492831 2023-09-06T20:34:53,005 WARN [Listener at jenkins-hbase3.apache.org/45863 {}] util.BackupUtils(373): Skip log file (can't parse): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/oldWALs/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032435590 2023-09-06T20:34:53,005 WARN [Listener at jenkins-hbase3.apache.org/45863 {}] util.BackupUtils(373): Skip log file (can't parse): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/oldWALs/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032479307 2023-09-06T20:34:53,415 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] util.BackupUtils(146): Attempting to copy table info for:ns2:test-16940324422881 target: hdfs://localhost:38135/backupUT/backup_1694032490651/ns2/test-16940324422881 descriptor: 'ns2:test-16940324422881', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:34:53,415 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-09-06T20:34:53,417 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] util.BackupUtils(151): Starting to write region info for table ns2:test-16940324422881 2023-09-06T20:34:53,823 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] util.BackupUtils(158): Finished writing region info for table ns2:test-16940324422881 2023-09-06T20:34:54,236 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] util.BackupUtils(146): Attempting to copy table info for:test-1694032442288 target: hdfs://localhost:38135/backupUT/backup_1694032490651/default/test-1694032442288 descriptor: 'test-1694032442288', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:34:54,236 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-09-06T20:34:54,238 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] util.BackupUtils(151): Starting to write region info for table test-1694032442288 2023-09-06T20:34:54,644 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] util.BackupUtils(158): Finished writing region info for table test-1694032442288 2023-09-06T20:34:54,651 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:38135/backupUT/.tmp/backup_1694032490651 from hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032457863;hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032479287;hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032457884 2023-09-06T20:34:54,657 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4af71113, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:54,658 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:34:54,659 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42048, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:34:54,661 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-09-06T20:34:54,661 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1694032442288 2023-09-06T20:34:54,664 DEBUG [hconnection-0x498c1eb6-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:54,666 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41558, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:54,668 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1694032442288: test-1694032442288; 2023-09-06T20:34:54,668 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16940324422881 2023-09-06T20:34:54,669 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16940324422881: ns2:test-16940324422881; 2023-09-06T20:34:54,670 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.HFileOutputFormat2(694): Configuring 2 reduce partitions to match current region count for all tables 2023-09-06T20:34:54,670 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/hbase-staging/partitions_1dac8e70-e3b0-49c2-9c6a-335d305b968a 2023-09-06T20:34:55,077 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:55,078 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:55,078 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:55,079 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:56,050 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/hadoop-2287929009041393038.jar 2023-09-06T20:34:56,051 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:56,051 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:56,052 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:56,052 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:56,052 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:56,053 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:56,053 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:56,053 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:56,054 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:56,055 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:34:56,056 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:34:56,056 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:34:56,056 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:34:56,057 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:34:56,057 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:34:56,058 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:34:56,058 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:34:56,059 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:34:56,059 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:34:56,059 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:34:56,060 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:34:56,061 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:56,061 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:56,062 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:56,062 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:56,062 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:56,063 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:34:56,063 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:34:56,064 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: default:test-1694032442288,ns2:test-16940324422881 2023-09-06T20:34:56,064 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:56,065 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:34:56,448 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region a44b05248e75fb3972b6ec9a34d9f4b8 changed from -1.0 to 0.0, refreshing cache 2023-09-06T20:34:57,147 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/hadoop-4318728648759519206.jar 2023-09-06T20:34:57,172 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032457863 for WAL files 2023-09-06T20:34:57,173 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032457863 2023-09-06T20:34:57,174 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032479287 for WAL files 2023-09-06T20:34:57,174 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032479287 2023-09-06T20:34:57,175 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032457884 for WAL files 2023-09-06T20:34:57,176 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032457884 2023-09-06T20:34:57,261 WARN [Listener at jenkins-hbase3.apache.org/45863 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/mapreduce.cluster.local.dir/1694032497220/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:34:57,262 WARN [Listener at jenkins-hbase3.apache.org/45863 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/mapreduce.cluster.local.dir/1694032497220/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:34:57,361 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032457863 for hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032457863 (-9223372036854775808:9223372036854775807) length:18860 2023-09-06T20:34:57,370 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-09-06T20:34:57,370 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-09-06T20:34:57,395 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032479287 for hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032479287 (-9223372036854775808:9223372036854775807) length:3657 2023-09-06T20:34:57,399 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-09-06T20:34:57,399 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-09-06T20:34:57,424 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032457884 for hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032457884 (-9223372036854775808:9223372036854775807) length:1198 2023-09-06T20:34:57,427 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-09-06T20:34:57,427 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-09-06T20:34:57,460 DEBUG [pool-248-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6575c554, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:57,465 DEBUG [hconnection-0x51ff428e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:57,467 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41568, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:57,469 DEBUG [pool-248-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:57,470 DEBUG [pool-248-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-t20] 2023-09-06T20:34:57,470 DEBUG [pool-248-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-09-06T20:34:57,475 DEBUG [pool-248-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/backupUT/.tmp/backup_1694032490651/_temporary/0/_temporary/attempt_local1114186174_0005_r_000000_0/ns2/test-16940324422881/f/1e0faca261184e28a258a305231a4702 is 38, key is row-t20/f:q1/1694032468944/Put/seqid=0 2023-09-06T20:34:57,476 WARN [Thread-1077 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:44489] Specified favored nodes: [jenkins-hbase3.apache.org:44489] 2023-09-06T20:34:57,910 DEBUG [pool-248-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59273ad7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:34:57,912 DEBUG [hconnection-0x7748a513-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:34:57,914 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41580, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:34:57,916 DEBUG [pool-248-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:34:57,917 DEBUG [pool-248-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-test-1694032442288-1-0] 2023-09-06T20:34:57,917 DEBUG [pool-248-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-09-06T20:34:57,928 DEBUG [pool-248-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/backupUT/.tmp/backup_1694032490651/_temporary/0/_temporary/attempt_local1114186174_0005_r_000001_0/default/test-1694032442288/f/5e3cf206af314b449a8159eeb0d3d7a1 is 59, key is row-test-1694032442288-1-10/f:q1/1694032468833/Put/seqid=0 2023-09-06T20:34:57,930 WARN [Thread-1087 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:44489] Specified favored nodes: [jenkins-hbase3.apache.org:44489] 2023-09-06T20:34:59,346 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:38135/backupUT 2023-09-06T20:34:59,346 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1694032490651 2023-09-06T20:34:59,346 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-09-06T20:34:59,357 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:38135/backupUT/.tmp/backup_1694032490651, hdfs://localhost:38135/backupUT] 2023-09-06T20:35:00,002 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-09-06T20:35:00,004 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1694032490651 2023-09-06T20:35:00,004 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 16005 bytes copied." 2023-09-06T20:35:00,004 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local987331493_0006 completed: true true 2023-09-06T20:35:00,007 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=4446445 FILE: Number of bytes written=7506557 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=299479 HDFS: Number of bytes written=2422565 HDFS: Number of read operations=1010 HDFS: Number of large read operations=0 HDFS: Number of write operations=326 Map-Reduce Framework Map input records=10 Map output records=0 Input split bytes=341 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1546649600 File Input Format Counters Bytes Read=2444 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=16005 Bytes Expected=16005 Files Copied=3 DIR_COPY=7 2023-09-06T20:35:00,007 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:38135/backupUT/.tmp/backup_1694032490651 to hdfs://localhost:38135/backupUT finished. 2023-09-06T20:35:00,012 ERROR [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(228): Unexpected Exception : Failed stage stage_3 in testing java.io.IOException: Failed stage stage_3 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:150) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:35:00,012 ERROR [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(240): BackupId=backup_1694032490651,startts=1694032492793,failedts=1694032500012,failedphase=INCREMENTAL_COPY,failedmessage=Failed stage stage_3 in testing 2023-09-06T20:35:00,013 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-09-06T20:35:00,017 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable backup:system 2023-09-06T20:35:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=66, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-09-06T20:35:00,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-09-06T20:35:00,021 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032500021"}]},"ts":"1694032500021"} 2023-09-06T20:35:00,023 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-09-06T20:35:00,024 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-09-06T20:35:00,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, UNASSIGN}] 2023-09-06T20:35:00,028 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, UNASSIGN 2023-09-06T20:35:00,028 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=cc8a9b52139566dcd11c25f6d6fdfaad, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:35:00,030 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE; CloseRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad, server=jenkins-hbase3.apache.org,44489,1694032433466}] 2023-09-06T20:35:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-09-06T20:35:00,182 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] handler.UnassignRegionHandler(119): Close cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:35:00,182 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1668): Closing cc8a9b52139566dcd11c25f6d6fdfaad, disabling compactions & flushes 2023-09-06T20:35:00,182 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1690): Closing region backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:35:00,182 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:35:00,183 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. after waiting 0 ms 2023-09-06T20:35:00,183 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:35:00,183 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(2819): Flushing cc8a9b52139566dcd11c25f6d6fdfaad 2/2 column families, dataSize=872 B heapSize=1.71 KB 2023-09-06T20:35:00,205 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/meta/16163d8a00194ca4885f00ec31ec05d8 is 171, key is trslm:hdfs://localhost:38135/backupUT\x00ns2:test-16940324422881/meta:log-roll-map/1694032500011/Put/seqid=0 2023-09-06T20:35:00,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-09-06T20:35:00,610 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=444 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/meta/16163d8a00194ca4885f00ec31ec05d8 2023-09-06T20:35:00,620 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/e86cdf79fe554284a4929c5d26a0831b is 218, key is session:backup_1694032490651/session:context/1694032500003/Put/seqid=0 2023-09-06T20:35:00,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-09-06T20:35:01,024 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=428 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/e86cdf79fe554284a4929c5d26a0831b 2023-09-06T20:35:01,030 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/meta/16163d8a00194ca4885f00ec31ec05d8 as hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/16163d8a00194ca4885f00ec31ec05d8 2023-09-06T20:35:01,036 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/16163d8a00194ca4885f00ec31ec05d8, entries=3, sequenceid=56, filesize=5.6 K 2023-09-06T20:35:01,037 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/e86cdf79fe554284a4929c5d26a0831b as hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/e86cdf79fe554284a4929c5d26a0831b 2023-09-06T20:35:01,043 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/e86cdf79fe554284a4929c5d26a0831b, entries=1, sequenceid=56, filesize=5.2 K 2023-09-06T20:35:01,043 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(3022): Finished flush of dataSize ~872 B/872, heapSize ~1.68 KB/1720, currentSize=0 B/0 for cc8a9b52139566dcd11c25f6d6fdfaad in 860ms, sequenceid=56, compaction requested=true 2023-09-06T20:35:01,052 DEBUG [StoreCloser-backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/f4a9d1ee08d64003a14895000c52e7db, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/ae2e5dd8863b45f491fb822de06cae0f, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/991095229826401eb83b47fa755c059c] to archive 2023-09-06T20:35:01,053 DEBUG [StoreCloser-backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-06T20:35:01,057 DEBUG [StoreCloser-backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/f4a9d1ee08d64003a14895000c52e7db to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/archive/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/f4a9d1ee08d64003a14895000c52e7db 2023-09-06T20:35:01,058 DEBUG [StoreCloser-backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/ae2e5dd8863b45f491fb822de06cae0f to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/archive/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/ae2e5dd8863b45f491fb822de06cae0f 2023-09-06T20:35:01,059 DEBUG [StoreCloser-backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/991095229826401eb83b47fa755c059c to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/archive/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/991095229826401eb83b47fa755c059c 2023-09-06T20:35:01,096 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=42 2023-09-06T20:35:01,096 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:35:01,097 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1909): Closed backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:35:01,097 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1622): Region close journal for cc8a9b52139566dcd11c25f6d6fdfaad: 2023-09-06T20:35:01,098 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] handler.UnassignRegionHandler(163): Closed cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:35:01,099 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=cc8a9b52139566dcd11c25f6d6fdfaad, regionState=CLOSED 2023-09-06T20:35:01,101 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=68, resume processing ppid=67 2023-09-06T20:35:01,101 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=68, ppid=67, state=SUCCESS; CloseRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad, server=jenkins-hbase3.apache.org,44489,1694032433466 in 1.0700 sec 2023-09-06T20:35:01,102 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=67, resume processing ppid=66 2023-09-06T20:35:01,102 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=67, ppid=66, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, UNASSIGN in 1.0760 sec 2023-09-06T20:35:01,103 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032501103"}]},"ts":"1694032501103"} 2023-09-06T20:35:01,104 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-09-06T20:35:01,106 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-09-06T20:35:01,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=66, state=SUCCESS; DisableTableProcedure table=backup:system in 1.0880 sec 2023-09-06T20:35:01,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-09-06T20:35:01,133 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1694032501132 table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:35:01,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032501133 (current time:1694032501133). 2023-09-06T20:35:01,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:35:01,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1694032501132 VERSION not specified, setting to 2 2023-09-06T20:35:01,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2300d656, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:01,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:01,142 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:59418, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:01,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:35:01,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=69, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032501132 table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:35:01,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032501132 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 69 2023-09-06T20:35:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=69 2023-09-06T20:35:01,145 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032501132 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:35:01,146 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032501132 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:35:01,148 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032501132 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:35:01,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=69 2023-09-06T20:35:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=69 2023-09-06T20:35:01,554 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032501132 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-09-06T20:35:01,555 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-09-06T20:35:01,555 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-09-06T20:35:01,555 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:01,557 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/16163d8a00194ca4885f00ec31ec05d8, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/89e49a43114a459a838964cbd810a62a] hfiles 2023-09-06T20:35:01,557 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/2): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/16163d8a00194ca4885f00ec31ec05d8 2023-09-06T20:35:01,557 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/2): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/89e49a43114a459a838964cbd810a62a 2023-09-06T20:35:01,557 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:01,559 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/22fa315304564a1397a14e4ba5bc21c8, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/7772e4890a534e54a647e460587efe32, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/e86cdf79fe554284a4929c5d26a0831b] hfiles 2023-09-06T20:35:01,559 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/3): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/22fa315304564a1397a14e4ba5bc21c8 2023-09-06T20:35:01,559 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/3): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/7772e4890a534e54a647e460587efe32 2023-09-06T20:35:01,559 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/3): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/e86cdf79fe554284a4929c5d26a0831b 2023-09-06T20:35:01,560 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-09-06T20:35:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=69 2023-09-06T20:35:01,965 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => cc8a9b52139566dcd11c25f6d6fdfaad, NAME => 'backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-09-06T20:35:01,966 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032501132 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:35:01,967 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032501132 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:35:01,967 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1694032501132 2023-09-06T20:35:01,968 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1694032501132 2023-09-06T20:35:02,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=69 2023-09-06T20:35:02,381 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032501132 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:35:02,389 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032501132 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:35:02,389 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1694032501132 to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1694032501132 2023-09-06T20:35:02,390 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032501132 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:35:02,390 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032501132 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 69 2023-09-06T20:35:02,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=69, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032501132 table=backup:system type=FLUSH ttl=0 } in 1.2470 sec 2023-09-06T20:35:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=69 2023-09-06T20:35:03,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=70, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032490675 type: FLUSH version: 2 ttl: 0 ) 2023-09-06T20:35:03,261 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(1015): Restore snapshot=snapshot_backup_system as table=backup:system 2023-09-06T20:35:03,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=70 2023-09-06T20:35:03,274 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/.tabledesc/.tableinfo.0000000003.635 2023-09-06T20:35:03,274 INFO [PEWorker-4 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/.tabledesc/.tableinfo.0000000004.635 2023-09-06T20:35:03,275 INFO [PEWorker-4 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:35:03,278 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032490675 type: FLUSH version: 2 ttl: 0 2023-09-06T20:35:03,278 DEBUG [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system 2023-09-06T20:35:03,280 DEBUG [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-09-06T20:35:03,280 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(227): region to restore: cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:35:03,286 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/16163d8a00194ca4885f00ec31ec05d8 to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/archive/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/16163d8a00194ca4885f00ec31ec05d8 2023-09-06T20:35:03,286 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:03,289 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/e86cdf79fe554284a4929c5d26a0831b to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/archive/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/e86cdf79fe554284a4929c5d26a0831b 2023-09-06T20:35:03,289 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:03,290 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032490675 type: FLUSH version: 2 ttl: 0 2023-09-06T20:35:03,291 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1694032503290"}]},"ts":"9223372036854775807"} 2023-09-06T20:35:03,292 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-06T20:35:03,293 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => cc8a9b52139566dcd11c25f6d6fdfaad, NAME => 'backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:35:03,293 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1694032503291"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032503291"}]},"ts":"1694032503291"} 2023-09-06T20:35:03,294 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:35:03,294 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-09-06T20:35:03,294 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => cc8a9b52139566dcd11c25f6d6fdfaad, NAME => 'backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:35:03,294 INFO [PEWorker-4 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-09-06T20:35:03,296 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=70, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032490675 type: FLUSH version: 2 ttl: 0 ) in 35 msec 2023-09-06T20:35:03,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=70 2023-09-06T20:35:03,364 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1694032501132" 2023-09-06T20:35:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(378): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1694032501132 2023-09-06T20:35:03,369 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.HMaster$11(2631): Client=jenkins//172.31.12.81 enable backup:system 2023-09-06T20:35:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=71, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-09-06T20:35:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=71 2023-09-06T20:35:03,373 INFO [PEWorker-2 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-09-06T20:35:03,373 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032503373"}]},"ts":"1694032503373"} 2023-09-06T20:35:03,374 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-09-06T20:35:03,376 DEBUG [PEWorker-2 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-09-06T20:35:03,377 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, ASSIGN}] 2023-09-06T20:35:03,378 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, ASSIGN 2023-09-06T20:35:03,379 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-09-06T20:35:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=71 2023-09-06T20:35:03,531 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=72 updating hbase:meta row=cc8a9b52139566dcd11c25f6d6fdfaad, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:35:03,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE; OpenRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad, server=jenkins-hbase3.apache.org,44489,1694032433466}] 2023-09-06T20:35:03,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=71 2023-09-06T20:35:03,688 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(130): Open backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:35:03,688 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7254): Opening region: {ENCODED => cc8a9b52139566dcd11c25f6d6fdfaad, NAME => 'backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:35:03,689 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:35:03,689 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:35:03,689 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(888): Instantiated backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:03,689 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7296): checking encryption for cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:35:03,689 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7299): checking classloading for cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:35:03,690 INFO [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:35:03,691 INFO [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc8a9b52139566dcd11c25f6d6fdfaad columnFamilyName meta 2023-09-06T20:35:03,691 DEBUG [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:03,698 DEBUG [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/89e49a43114a459a838964cbd810a62a 2023-09-06T20:35:03,698 INFO [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.HStore(324): Store=cc8a9b52139566dcd11c25f6d6fdfaad/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:03,698 INFO [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:35:03,699 INFO [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc8a9b52139566dcd11c25f6d6fdfaad columnFamilyName session 2023-09-06T20:35:03,699 DEBUG [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:03,711 DEBUG [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/22fa315304564a1397a14e4ba5bc21c8 2023-09-06T20:35:03,716 DEBUG [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/7772e4890a534e54a647e460587efe32 2023-09-06T20:35:03,716 INFO [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.HStore(324): Store=cc8a9b52139566dcd11c25f6d6fdfaad/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:03,717 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:35:03,718 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:35:03,719 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-09-06T20:35:03,720 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1076): writing seq id for cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:35:03,721 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): Opened cc8a9b52139566dcd11c25f6d6fdfaad; next sequenceid=60; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9805055040, jitterRate=-0.08683308959007263}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-09-06T20:35:03,723 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(992): Region open journal for cc8a9b52139566dcd11c25f6d6fdfaad: 2023-09-06T20:35:03,723 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad., pid=73, masterSystemTime=1694032503684 2023-09-06T20:35:03,725 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:35:03,725 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(158): Opened backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:35:03,725 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=72 updating hbase:meta row=cc8a9b52139566dcd11c25f6d6fdfaad, regionState=OPEN, openSeqNum=60, regionLocation=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:35:03,728 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=73, resume processing ppid=72 2023-09-06T20:35:03,728 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=73, ppid=72, state=SUCCESS; OpenRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad, server=jenkins-hbase3.apache.org,44489,1694032433466 in 194 msec 2023-09-06T20:35:03,730 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=72, resume processing ppid=71 2023-09-06T20:35:03,730 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=72, ppid=71, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, ASSIGN in 352 msec 2023-09-06T20:35:03,730 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032503730"}]},"ts":"1694032503730"} 2023-09-06T20:35:03,732 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-09-06T20:35:03,733 INFO [PEWorker-5 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-09-06T20:35:03,735 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=71, state=SUCCESS; EnableTableProcedure table=backup:system in 365 msec 2023-09-06T20:35:03,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=71 2023-09-06T20:35:03,976 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(1525): Done restoring backup system table 2023-09-06T20:35:03,976 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:35:03,981 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:35:03,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:35:03,984 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:35:03,984 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: INCREMENTAL_COPY 2023-09-06T20:35:03,985 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:38135/backupUT/backup_1694032490651/ns2/test-16940324422881 done. 2023-09-06T20:35:03,985 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(208): hdfs://localhost:38135/backupUT/backup_1694032490651/ns2 is empty, remove it. 2023-09-06T20:35:03,986 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:38135/backupUT/backup_1694032490651/default/test-1694032442288 done. 2023-09-06T20:35:03,987 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(208): hdfs://localhost:38135/backupUT/backup_1694032490651/default is empty, remove it. 2023-09-06T20:35:03,988 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:35:03,996 ERROR [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(246): Backup backup_1694032490651 failed. Backup session finished. Status: FAILURE 2023-09-06T20:35:03,997 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:03,997 ERROR [Listener at jenkins-hbase3.apache.org/45863 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_3 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:168) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_3 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:150) ~[test-classes/:?] ... 41 more 2023-09-06T20:35:04,006 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] backup.TestIncrementalBackupWithFailures(136): Running stage 4 2023-09-06T20:35:04,029 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c971c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:04,099 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:35:04,101 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42754, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:35:04,105 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:04,107 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:59430, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:04,152 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:35:04,156 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:35:04,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032504156 (current time:1694032504156). 2023-09-06T20:35:04,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:35:04,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:35:04,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f6bc019, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:04,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:04,165 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:59442, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:04,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:04,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:35:04,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=74, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:35:04,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 74 2023-09-06T20:35:04,168 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:35:04,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=74 2023-09-06T20:35:04,169 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:35:04,171 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:35:04,177 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:35:04,177 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE; SnapshotRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad}] 2023-09-06T20:35:04,179 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=75, ppid=74, state=RUNNABLE; SnapshotRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:35:04,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=74 2023-09-06T20:35:04,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44489 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=75 2023-09-06T20:35:04,330 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:35:04,332 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.HRegion(2819): Flushing cc8a9b52139566dcd11c25f6d6fdfaad 2/2 column families, dataSize=341 B heapSize=1.05 KB 2023-09-06T20:35:04,355 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/c34ffa13d15949f6980d3190977d1e4a is 256, key is session:backup_1694032490651/session:context/1694032503988/Put/seqid=0 2023-09-06T20:35:04,363 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=341 B at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/c34ffa13d15949f6980d3190977d1e4a 2023-09-06T20:35:04,369 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/c34ffa13d15949f6980d3190977d1e4a as hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/c34ffa13d15949f6980d3190977d1e4a 2023-09-06T20:35:04,374 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/c34ffa13d15949f6980d3190977d1e4a, entries=2, sequenceid=65, filesize=5.2 K 2023-09-06T20:35:04,375 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.HRegion(3022): Finished flush of dataSize ~341 B/341, heapSize ~808 B/808, currentSize=0 B/0 for cc8a9b52139566dcd11c25f6d6fdfaad in 43ms, sequenceid=65, compaction requested=true 2023-09-06T20:35:04,375 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-09-06T20:35:04,376 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.HRegion(2520): Flush status journal for cc8a9b52139566dcd11c25f6d6fdfaad: 2023-09-06T20:35:04,376 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. for snapshot_backup_system completed. 2023-09-06T20:35:04,376 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:35:04,376 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:35:04,376 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/89e49a43114a459a838964cbd810a62a] hfiles 2023-09-06T20:35:04,376 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/89e49a43114a459a838964cbd810a62a for snapshot=snapshot_backup_system 2023-09-06T20:35:04,376 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/22fa315304564a1397a14e4ba5bc21c8, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/7772e4890a534e54a647e460587efe32, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/c34ffa13d15949f6980d3190977d1e4a] hfiles 2023-09-06T20:35:04,376 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/22fa315304564a1397a14e4ba5bc21c8 for snapshot=snapshot_backup_system 2023-09-06T20:35:04,376 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/7772e4890a534e54a647e460587efe32 for snapshot=snapshot_backup_system 2023-09-06T20:35:04,377 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/c34ffa13d15949f6980d3190977d1e4a for snapshot=snapshot_backup_system 2023-09-06T20:35:04,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=74 2023-09-06T20:35:04,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=74 2023-09-06T20:35:04,786 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:35:04,786 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=75 2023-09-06T20:35:04,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.HMaster(3995): Remote procedure done, pid=75 2023-09-06T20:35:04,786 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:35:04,788 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=75, ppid=74, state=RUNNABLE; SnapshotRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:35:04,790 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=75, resume processing ppid=74 2023-09-06T20:35:04,790 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:35:04,790 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=75, ppid=74, state=SUCCESS; SnapshotRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad in 612 msec 2023-09-06T20:35:04,790 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:35:04,791 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:35:04,791 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:35:04,792 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:35:04,803 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:35:04,810 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:35:04,811 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:35:04,812 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:35:04,812 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 74 2023-09-06T20:35:04,813 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=74, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 646 msec 2023-09-06T20:35:05,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=74 2023-09-06T20:35:05,274 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(115): Backup backup_1694032504130 started at 1694032505274. 2023-09-06T20:35:05,276 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(119): Backup session backup_1694032504130 has been started. 2023-09-06T20:35:05,277 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] backup.TestBackupBase$IncrementalTableBackupClientForTest(128): For incremental backup, current table set is [test-1694032442288, ns2:test-16940324422881] 2023-09-06T20:35:05,279 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager(76): StartCode 1694032437629for backupID backup_1694032504130 2023-09-06T20:35:05,279 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-09-06T20:35:05,280 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-09-06T20:35:05,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-09-06T20:35:05,280 INFO [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-09-06T20:35:05,280 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-09-06T20:35:05,282 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-09-06T20:35:05,282 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-09-06T20:35:05,282 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:35:05,283 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:35:05,283 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:35:05,283 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:35:05,284 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-09-06T20:35:05,284 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-09-06T20:35:05,284 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-09-06T20:35:05,284 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:35:05,284 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-09-06T20:35:05,285 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-09-06T20:35:05,285 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-09-06T20:35:05,285 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-09-06T20:35:05,285 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-09-06T20:35:05,286 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-09-06T20:35:05,286 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-09-06T20:35:05,286 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,44489,1694032433466' joining acquired barrier for procedure (rolllog) in zk 2023-09-06T20:35:05,288 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-09-06T20:35:05,288 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:35:05,288 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:35:05,288 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:35:05,289 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-09-06T20:35:05,289 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:35:05,289 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:35:05,289 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:35:05,289 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:35:05,290 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:35:05,290 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,44489,1694032433466' joining acquired barrier for procedure 'rolllog' on coordinator 2023-09-06T20:35:05,290 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@1d75c7c6[Count = 0] remaining members to acquire global barrier 2023-09-06T20:35:05,290 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-09-06T20:35:05,290 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-09-06T20:35:05,292 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-09-06T20:35:05,292 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-09-06T20:35:05,292 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-09-06T20:35:05,292 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-09-06T20:35:05,293 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-09-06T20:35:05,293 DEBUG [rs(jenkins-hbase3.apache.org,44489,1694032433466)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:35:05,293 INFO [rs(jenkins-hbase3.apache.org,44489,1694032433466)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1694032492814 highest: 1694032492831 on jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:35:05,293 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0:(num 1694032492814) roll requested 2023-09-06T20:35:05,304 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032505293, exclude list is [], retry=0 2023-09-06T20:35:05,310 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK] 2023-09-06T20:35:05,312 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032492814 with entries=13, filesize=3.55 KB; new WAL /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032505293 2023-09-06T20:35:05,312 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK]] 2023-09-06T20:35:05,312 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032492814 is not closed yet, will try archiving it next time 2023-09-06T20:35:05,312 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1:(num 1694032492831) roll requested 2023-09-06T20:35:05,312 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032479287 to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/oldWALs/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032479287 2023-09-06T20:35:05,323 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032505312, exclude list is [], retry=0 2023-09-06T20:35:05,327 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK] 2023-09-06T20:35:05,328 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032492831 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032505312 2023-09-06T20:35:05,328 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK]] 2023-09-06T20:35:05,328 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032492831 is not closed yet, will try archiving it next time 2023-09-06T20:35:05,328 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta:.meta(num 1694032492847) roll requested 2023-09-06T20:35:05,330 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032492831 not finished, retry = 0 2023-09-06T20:35:05,340 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta.1694032505328.meta, exclude list is [], retry=0 2023-09-06T20:35:05,343 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK] 2023-09-06T20:35:05,344 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta.1694032492847.meta with entries=10, filesize=2.50 KB; new WAL /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta.1694032505328.meta 2023-09-06T20:35:05,344 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:37183,DS-edff5bfa-143b-471c-9d7d-03661acff561,DISK]] 2023-09-06T20:35:05,344 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta.1694032492847.meta is not closed yet, will try archiving it next time 2023-09-06T20:35:05,353 DEBUG [rs(jenkins-hbase3.apache.org,44489,1694032433466)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2023-09-06T20:35:05,353 INFO [rs(jenkins-hbase3.apache.org,44489,1694032433466)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1694032505293 on jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:35:05,360 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-09-06T20:35:05,360 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,44489,1694032433466' in zk 2023-09-06T20:35:05,370 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-09-06T20:35:05,370 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:35:05,370 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:35:05,370 DEBUG [member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-09-06T20:35:05,370 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:35:05,371 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:35:05,371 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:35:05,372 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:35:05,372 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:35:05,372 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:35:05,372 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:35:05,373 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:35:05,373 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:35:05,373 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,44489,1694032433466': 2023-09-06T20:35:05,374 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,44489,1694032433466' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-09-06T20:35:05,374 INFO [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-09-06T20:35:05,374 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-09-06T20:35:05,374 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-09-06T20:35:05,374 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-09-06T20:35:05,374 INFO [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-09-06T20:35:05,375 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-09-06T20:35:05,375 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-09-06T20:35:05,375 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:35:05,375 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:35:05,375 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:35:05,376 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:35:05,376 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:35:05,376 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:35:05,376 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:35:05,376 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:35:05,377 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:35:05,377 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:35:05,377 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:35:05,378 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:35:05,378 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:35:05,378 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:35:05,383 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:35:05,383 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:35:05,383 DEBUG [(jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:35:05,383 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-09-06T20:35:05,383 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-09-06T20:35:05,384 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:35:05,384 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:35:05,431 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032492831 to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/oldWALs/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032492831 2023-09-06T20:35:05,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-09-06T20:35:05,487 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:44489=1694032437629} newestTimestamps: {jenkins-hbase3.apache.org:44489=1694032492831} 2023-09-06T20:35:05,492 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta.1694032435949.meta 2023-09-06T20:35:05,493 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta.1694032457903.meta 2023-09-06T20:35:05,493 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta.1694032479323.meta 2023-09-06T20:35:05,493 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta.1694032492847.meta 2023-09-06T20:35:05,493 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta.1694032505328.meta 2023-09-06T20:35:05,493 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032457863 2023-09-06T20:35:05,493 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032492814 2023-09-06T20:35:05,493 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032505293 2023-09-06T20:35:05,493 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032457884 2023-09-06T20:35:05,493 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032505312 2023-09-06T20:35:05,494 WARN [Listener at jenkins-hbase3.apache.org/45863 {}] util.BackupUtils(373): Skip log file (can't parse): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/oldWALs/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032435590 2023-09-06T20:35:05,494 WARN [Listener at jenkins-hbase3.apache.org/45863 {}] util.BackupUtils(373): Skip log file (can't parse): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/oldWALs/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032479287 2023-09-06T20:35:05,494 WARN [Listener at jenkins-hbase3.apache.org/45863 {}] util.BackupUtils(373): Skip log file (can't parse): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/oldWALs/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032479307 2023-09-06T20:35:05,495 WARN [Listener at jenkins-hbase3.apache.org/45863 {}] util.BackupUtils(373): Skip log file (can't parse): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/oldWALs/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032492831 2023-09-06T20:35:05,906 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] util.BackupUtils(146): Attempting to copy table info for:ns2:test-16940324422881 target: hdfs://localhost:38135/backupUT/backup_1694032504130/ns2/test-16940324422881 descriptor: 'ns2:test-16940324422881', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:35:05,906 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-09-06T20:35:05,908 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] util.BackupUtils(151): Starting to write region info for table ns2:test-16940324422881 2023-09-06T20:35:06,314 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] util.BackupUtils(158): Finished writing region info for table ns2:test-16940324422881 2023-09-06T20:35:06,654 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:35:06,725 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] util.BackupUtils(146): Attempting to copy table info for:test-1694032442288 target: hdfs://localhost:38135/backupUT/backup_1694032504130/default/test-1694032442288 descriptor: 'test-1694032442288', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:35:06,725 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-09-06T20:35:06,728 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] util.BackupUtils(151): Starting to write region info for table test-1694032442288 2023-09-06T20:35:06,737 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] util.BackupUtils(158): Finished writing region info for table test-1694032442288 2023-09-06T20:35:06,744 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:38135/backupUT/.tmp/backup_1694032504130 from hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032457863;hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032492814;hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032457884 2023-09-06T20:35:06,762 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6afbc060, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:06,778 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:35:06,781 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42756, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:35:06,784 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-09-06T20:35:06,784 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1694032442288 2023-09-06T20:35:06,789 DEBUG [hconnection-0x31c250e0-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:06,790 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:59452, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:06,792 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1694032442288: test-1694032442288; 2023-09-06T20:35:06,792 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16940324422881 2023-09-06T20:35:06,795 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16940324422881: ns2:test-16940324422881; 2023-09-06T20:35:06,795 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.HFileOutputFormat2(694): Configuring 2 reduce partitions to match current region count for all tables 2023-09-06T20:35:06,795 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/hbase-staging/partitions_942e8690-9b45-43b3-96b8-0b950716e13c 2023-09-06T20:35:07,210 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:07,210 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:07,211 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:07,211 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:08,220 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/hadoop-5449070721200005201.jar 2023-09-06T20:35:08,220 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:08,221 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:08,221 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:08,221 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:08,222 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:08,222 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:08,223 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:08,223 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:08,223 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:08,224 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:35:08,225 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:35:08,225 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:35:08,225 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:35:08,226 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:35:08,226 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:35:08,227 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:35:08,227 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:35:08,227 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:35:08,228 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:35:08,228 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:35:08,228 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:35:08,229 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:08,229 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:08,230 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:08,230 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:08,230 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:08,231 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:35:08,231 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:35:08,231 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: default:test-1694032442288,ns2:test-16940324422881 2023-09-06T20:35:08,232 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:08,233 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:35:09,254 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/hadoop-3613537800475388165.jar 2023-09-06T20:35:09,275 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032457863 for WAL files 2023-09-06T20:35:09,276 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032457863 2023-09-06T20:35:09,276 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032492814 for WAL files 2023-09-06T20:35:09,277 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032492814 2023-09-06T20:35:09,277 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032457884 for WAL files 2023-09-06T20:35:09,278 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032457884 2023-09-06T20:35:09,372 WARN [Listener at jenkins-hbase3.apache.org/45863 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/mapreduce.cluster.local.dir/1694032509330/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:35:09,372 WARN [Listener at jenkins-hbase3.apache.org/45863 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/mapreduce.cluster.local.dir/1694032509330/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:35:09,462 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032457863 for hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032457863 (-9223372036854775808:9223372036854775807) length:18860 2023-09-06T20:35:09,468 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-09-06T20:35:09,468 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-09-06T20:35:09,500 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032492814 for hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0.1694032492814 (-9223372036854775808:9223372036854775807) length:3644 2023-09-06T20:35:09,503 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-09-06T20:35:09,503 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-09-06T20:35:09,525 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032457884 for hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/WALs/jenkins-hbase3.apache.org,44489,1694032433466/jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1.1694032457884 (-9223372036854775808:9223372036854775807) length:1198 2023-09-06T20:35:09,528 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-09-06T20:35:09,528 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-09-06T20:35:09,548 DEBUG [pool-271-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36edb94b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:09,550 DEBUG [hconnection-0x3e35360d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:09,552 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:59456, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:09,553 DEBUG [pool-271-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:09,554 DEBUG [pool-271-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-t20] 2023-09-06T20:35:09,554 DEBUG [pool-271-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-09-06T20:35:09,557 DEBUG [pool-271-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/backupUT/.tmp/backup_1694032504130/_temporary/0/_temporary/attempt_local1882518732_0007_r_000000_0/ns2/test-16940324422881/f/03ed95c8a1fd417c890000f734541a9b is 38, key is row-t20/f:q1/1694032468944/Put/seqid=0 2023-09-06T20:35:09,558 WARN [Thread-1236 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:44489] Specified favored nodes: [jenkins-hbase3.apache.org:44489] 2023-09-06T20:35:09,581 DEBUG [pool-271-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@307bfb13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:09,584 DEBUG [hconnection-0x45bd5017-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:09,586 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:59458, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:09,588 DEBUG [pool-271-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:09,591 DEBUG [pool-271-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-test-1694032442288-1-0] 2023-09-06T20:35:09,591 DEBUG [pool-271-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-09-06T20:35:09,600 DEBUG [pool-271-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/backupUT/.tmp/backup_1694032504130/_temporary/0/_temporary/attempt_local1882518732_0007_r_000001_0/default/test-1694032442288/f/7f44bf8d4d9f48b28f9ff3ec27420f98 is 59, key is row-test-1694032442288-1-10/f:q1/1694032468833/Put/seqid=0 2023-09-06T20:35:09,601 WARN [Thread-1246 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:44489] Specified favored nodes: [jenkins-hbase3.apache.org:44489] 2023-09-06T20:35:10,415 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:38135/backupUT 2023-09-06T20:35:10,415 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1694032504130 2023-09-06T20:35:10,415 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-09-06T20:35:10,421 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:38135/backupUT/.tmp/backup_1694032504130, hdfs://localhost:38135/backupUT] 2023-09-06T20:35:11,526 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-09-06T20:35:11,527 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1694032504130 2023-09-06T20:35:11,528 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 16005 bytes copied." 2023-09-06T20:35:11,528 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local1046774060_0008 completed: true true 2023-09-06T20:35:11,531 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=5102042 FILE: Number of bytes written=9931726 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=394516 HDFS: Number of bytes written=2475177 HDFS: Number of read operations=1293 HDFS: Number of large read operations=0 HDFS: Number of write operations=413 Map-Reduce Framework Map input records=10 Map output records=0 Input split bytes=341 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1527250944 File Input Format Counters Bytes Read=2444 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=16005 Bytes Expected=16005 Files Copied=3 DIR_COPY=7 2023-09-06T20:35:11,531 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:38135/backupUT/.tmp/backup_1694032504130 to hdfs://localhost:38135/backupUT finished. 2023-09-06T20:35:11,542 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(433): deleted 0 original bulkload rows 2023-09-06T20:35:11,542 ERROR [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(228): Unexpected Exception : Failed stage stage_4 in testing java.io.IOException: Failed stage stage_4 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:160) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:35:11,543 ERROR [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(240): BackupId=backup_1694032504130,startts=1694032505274,failedts=1694032511543,failedphase=INCREMENTAL_COPY,failedmessage=Failed stage stage_4 in testing 2023-09-06T20:35:11,543 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-09-06T20:35:11,548 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable backup:system 2023-09-06T20:35:11,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=76, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-09-06T20:35:11,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=76 2023-09-06T20:35:11,552 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032511552"}]},"ts":"1694032511552"} 2023-09-06T20:35:11,553 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-09-06T20:35:11,555 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-09-06T20:35:11,556 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, UNASSIGN}] 2023-09-06T20:35:11,557 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=77, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, UNASSIGN 2023-09-06T20:35:11,558 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=77 updating hbase:meta row=cc8a9b52139566dcd11c25f6d6fdfaad, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:35:11,559 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE; CloseRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad, server=jenkins-hbase3.apache.org,44489,1694032433466}] 2023-09-06T20:35:11,560 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-09-06T20:35:11,560 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-09-06T20:35:11,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=76 2023-09-06T20:35:11,711 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] handler.UnassignRegionHandler(119): Close cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:35:11,711 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1668): Closing cc8a9b52139566dcd11c25f6d6fdfaad, disabling compactions & flushes 2023-09-06T20:35:11,711 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1690): Closing region backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:35:11,711 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:35:11,711 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. after waiting 0 ms 2023-09-06T20:35:11,711 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:35:11,711 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(2819): Flushing cc8a9b52139566dcd11c25f6d6fdfaad 2/2 column families, dataSize=959 B heapSize=1.87 KB 2023-09-06T20:35:11,725 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/meta/f0a722ab706945709d8a5602eb20c5e0 is 171, key is trslm:hdfs://localhost:38135/backupUT\x00ns2:test-16940324422881/meta:log-roll-map/1694032511534/Put/seqid=0 2023-09-06T20:35:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=76 2023-09-06T20:35:12,131 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=531 B at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/meta/f0a722ab706945709d8a5602eb20c5e0 2023-09-06T20:35:12,140 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/9df2dee3a04046ed8e114f0f42b023aa is 218, key is session:backup_1694032504130/session:context/1694032511526/Put/seqid=0 2023-09-06T20:35:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=76 2023-09-06T20:35:12,548 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=428 B at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/9df2dee3a04046ed8e114f0f42b023aa 2023-09-06T20:35:12,554 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/meta/f0a722ab706945709d8a5602eb20c5e0 as hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/f0a722ab706945709d8a5602eb20c5e0 2023-09-06T20:35:12,561 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/f0a722ab706945709d8a5602eb20c5e0, entries=4, sequenceid=73, filesize=5.7 K 2023-09-06T20:35:12,561 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/9df2dee3a04046ed8e114f0f42b023aa as hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/9df2dee3a04046ed8e114f0f42b023aa 2023-09-06T20:35:12,567 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/9df2dee3a04046ed8e114f0f42b023aa, entries=1, sequenceid=73, filesize=5.2 K 2023-09-06T20:35:12,568 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(3022): Finished flush of dataSize ~959 B/959, heapSize ~1.84 KB/1880, currentSize=0 B/0 for cc8a9b52139566dcd11c25f6d6fdfaad in 857ms, sequenceid=73, compaction requested=true 2023-09-06T20:35:12,579 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/recovered.edits/76.seqid, newMaxSeqId=76, maxSeqId=59 2023-09-06T20:35:12,579 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:35:12,580 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1909): Closed backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:35:12,580 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1622): Region close journal for cc8a9b52139566dcd11c25f6d6fdfaad: 2023-09-06T20:35:12,581 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] handler.UnassignRegionHandler(163): Closed cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:35:12,582 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=77 updating hbase:meta row=cc8a9b52139566dcd11c25f6d6fdfaad, regionState=CLOSED 2023-09-06T20:35:12,584 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=78, resume processing ppid=77 2023-09-06T20:35:12,584 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=78, ppid=77, state=SUCCESS; CloseRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad, server=jenkins-hbase3.apache.org,44489,1694032433466 in 1.0240 sec 2023-09-06T20:35:12,586 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=77, resume processing ppid=76 2023-09-06T20:35:12,586 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=77, ppid=76, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, UNASSIGN in 1.0280 sec 2023-09-06T20:35:12,586 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032512586"}]},"ts":"1694032512586"} 2023-09-06T20:35:12,587 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-09-06T20:35:12,589 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-09-06T20:35:12,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=76, state=SUCCESS; DisableTableProcedure table=backup:system in 1.0410 sec 2023-09-06T20:35:12,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=76 2023-09-06T20:35:12,664 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1694032512664 table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:35:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032512664 (current time:1694032512664). 2023-09-06T20:35:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:35:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1694032512664 VERSION not specified, setting to 2 2023-09-06T20:35:12,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3eb6132b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:35:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:35:12,673 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:40040, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:35:12,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:12,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:35:12,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=79, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032512664 table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:35:12,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032512664 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 79 2023-09-06T20:35:12,677 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032512664 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:35:12,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=79 2023-09-06T20:35:12,678 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032512664 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:35:12,680 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032512664 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:35:12,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=79 2023-09-06T20:35:12,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=79 2023-09-06T20:35:13,087 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032512664 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-09-06T20:35:13,088 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-09-06T20:35:13,088 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-09-06T20:35:13,088 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:13,090 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/89e49a43114a459a838964cbd810a62a, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/f0a722ab706945709d8a5602eb20c5e0] hfiles 2023-09-06T20:35:13,090 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/2): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/89e49a43114a459a838964cbd810a62a 2023-09-06T20:35:13,090 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/2): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/f0a722ab706945709d8a5602eb20c5e0 2023-09-06T20:35:13,090 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:13,092 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/22fa315304564a1397a14e4ba5bc21c8, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/7772e4890a534e54a647e460587efe32, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/9df2dee3a04046ed8e114f0f42b023aa, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/c34ffa13d15949f6980d3190977d1e4a] hfiles 2023-09-06T20:35:13,092 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/4): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/22fa315304564a1397a14e4ba5bc21c8 2023-09-06T20:35:13,092 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/4): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/7772e4890a534e54a647e460587efe32 2023-09-06T20:35:13,092 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/4): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/9df2dee3a04046ed8e114f0f42b023aa 2023-09-06T20:35:13,092 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (4/4): hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/c34ffa13d15949f6980d3190977d1e4a 2023-09-06T20:35:13,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=79 2023-09-06T20:35:13,498 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => cc8a9b52139566dcd11c25f6d6fdfaad, NAME => 'backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-09-06T20:35:13,499 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032512664 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:35:13,500 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032512664 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:35:13,500 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1694032512664 2023-09-06T20:35:13,500 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1694032512664 2023-09-06T20:35:13,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=79 2023-09-06T20:35:13,914 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032512664 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:35:13,922 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032512664 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:35:13,922 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1694032512664 to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1694032512664 2023-09-06T20:35:13,923 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032512664 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:35:13,923 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032512664 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 79 2023-09-06T20:35:13,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=79, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032512664 table=backup:system type=FLUSH ttl=0 } in 1.2480 sec 2023-09-06T20:35:14,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=79 2023-09-06T20:35:14,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=80, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032504156 type: FLUSH version: 2 ttl: 0 ) 2023-09-06T20:35:14,793 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(1015): Restore snapshot=snapshot_backup_system as table=backup:system 2023-09-06T20:35:14,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=80 2023-09-06T20:35:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=80 2023-09-06T20:35:15,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=80 2023-09-06T20:35:15,208 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/.tabledesc/.tableinfo.0000000004.635 2023-09-06T20:35:15,208 INFO [PEWorker-3 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/.tabledesc/.tableinfo.0000000005.635 2023-09-06T20:35:15,209 INFO [PEWorker-3 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:35:15,211 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032504156 type: FLUSH version: 2 ttl: 0 2023-09-06T20:35:15,211 DEBUG [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system 2023-09-06T20:35:15,214 DEBUG [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-09-06T20:35:15,214 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(227): region to restore: cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:35:15,221 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/f0a722ab706945709d8a5602eb20c5e0 to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/archive/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/f0a722ab706945709d8a5602eb20c5e0 2023-09-06T20:35:15,221 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:15,225 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/9df2dee3a04046ed8e114f0f42b023aa to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/archive/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/9df2dee3a04046ed8e114f0f42b023aa 2023-09-06T20:35:15,225 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:15,225 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032504156 type: FLUSH version: 2 ttl: 0 2023-09-06T20:35:15,226 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1694032515226"}]},"ts":"9223372036854775807"} 2023-09-06T20:35:15,228 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-06T20:35:15,229 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => cc8a9b52139566dcd11c25f6d6fdfaad, NAME => 'backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:35:15,229 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1694032515227"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032515227"}]},"ts":"1694032515227"} 2023-09-06T20:35:15,230 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:35:15,230 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-09-06T20:35:15,230 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => cc8a9b52139566dcd11c25f6d6fdfaad, NAME => 'backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:35:15,230 INFO [PEWorker-3 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-09-06T20:35:15,232 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=80, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032504156 type: FLUSH version: 2 ttl: 0 ) in 439 msec 2023-09-06T20:35:15,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=80 2023-09-06T20:35:15,400 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1694032512664" 2023-09-06T20:35:15,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(378): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1694032512664 2023-09-06T20:35:15,404 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.HMaster$11(2631): Client=jenkins//172.31.12.81 enable backup:system 2023-09-06T20:35:15,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] procedure2.ProcedureExecutor(1032): Stored pid=81, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-09-06T20:35:15,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=81 2023-09-06T20:35:15,409 INFO [PEWorker-4 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-09-06T20:35:15,409 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032515409"}]},"ts":"1694032515409"} 2023-09-06T20:35:15,410 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-09-06T20:35:15,413 DEBUG [PEWorker-4 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-09-06T20:35:15,413 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, ASSIGN}] 2023-09-06T20:35:15,415 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, ASSIGN 2023-09-06T20:35:15,416 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-09-06T20:35:15,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=81 2023-09-06T20:35:15,568 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=cc8a9b52139566dcd11c25f6d6fdfaad, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:35:15,569 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE; OpenRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad, server=jenkins-hbase3.apache.org,44489,1694032433466}] 2023-09-06T20:35:15,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=81 2023-09-06T20:35:15,724 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(130): Open backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:35:15,725 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7254): Opening region: {ENCODED => cc8a9b52139566dcd11c25f6d6fdfaad, NAME => 'backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:35:15,725 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:35:15,725 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:35:15,725 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(888): Instantiated backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:35:15,725 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7296): checking encryption for cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:35:15,725 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7299): checking classloading for cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:35:15,726 INFO [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:35:15,727 INFO [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc8a9b52139566dcd11c25f6d6fdfaad columnFamilyName meta 2023-09-06T20:35:15,727 DEBUG [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:15,733 DEBUG [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/meta/89e49a43114a459a838964cbd810a62a 2023-09-06T20:35:15,733 INFO [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.HStore(324): Store=cc8a9b52139566dcd11c25f6d6fdfaad/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:15,733 INFO [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:35:15,734 INFO [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc8a9b52139566dcd11c25f6d6fdfaad columnFamilyName session 2023-09-06T20:35:15,734 DEBUG [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:35:15,740 DEBUG [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/22fa315304564a1397a14e4ba5bc21c8 2023-09-06T20:35:15,745 DEBUG [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/7772e4890a534e54a647e460587efe32 2023-09-06T20:35:15,749 DEBUG [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/c34ffa13d15949f6980d3190977d1e4a 2023-09-06T20:35:15,749 INFO [StoreOpener-cc8a9b52139566dcd11c25f6d6fdfaad-1 {}] regionserver.HStore(324): Store=cc8a9b52139566dcd11c25f6d6fdfaad/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:35:15,750 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:35:15,751 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:35:15,753 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-09-06T20:35:15,754 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1076): writing seq id for cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:35:15,755 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1093): Opened cc8a9b52139566dcd11c25f6d6fdfaad; next sequenceid=77; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11343503680, jitterRate=0.05644610524177551}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-09-06T20:35:15,756 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(992): Region open journal for cc8a9b52139566dcd11c25f6d6fdfaad: 2023-09-06T20:35:15,757 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad., pid=83, masterSystemTime=1694032515721 2023-09-06T20:35:15,757 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.CompactSplit(403): Add compact mark for store cc8a9b52139566dcd11c25f6d6fdfaad:session, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:35:15,759 DEBUG [RS:0;jenkins-hbase3:44489-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:35:15,759 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:35:15,761 DEBUG [RS:0;jenkins-hbase3:44489-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 16834 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:35:15,762 DEBUG [RS:0;jenkins-hbase3:44489-longCompactions-0 {}] regionserver.HStore(1534): cc8a9b52139566dcd11c25f6d6fdfaad/session is initiating minor compaction (all files) 2023-09-06T20:35:15,762 INFO [RS:0;jenkins-hbase3:44489-longCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of cc8a9b52139566dcd11c25f6d6fdfaad/session in backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:35:15,762 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:35:15,762 INFO [RS:0;jenkins-hbase3:44489-longCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/22fa315304564a1397a14e4ba5bc21c8, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/7772e4890a534e54a647e460587efe32, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/c34ffa13d15949f6980d3190977d1e4a] into tmpdir=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp, totalSize=16.4 K 2023-09-06T20:35:15,762 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(158): Opened backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:35:15,762 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=cc8a9b52139566dcd11c25f6d6fdfaad, regionState=OPEN, openSeqNum=77, regionLocation=jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:35:15,763 DEBUG [RS:0;jenkins-hbase3:44489-longCompactions-0 {}] compactions.Compactor(223): Compacting 22fa315304564a1397a14e4ba5bc21c8, keycount=4, bloomtype=ROW, size=6.0 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1694032468780 2023-09-06T20:35:15,763 DEBUG [RS:0;jenkins-hbase3:44489-longCompactions-0 {}] compactions.Compactor(223): Compacting 7772e4890a534e54a647e460587efe32, keycount=2, bloomtype=ROW, size=5.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1694032490586 2023-09-06T20:35:15,764 DEBUG [RS:0;jenkins-hbase3:44489-longCompactions-0 {}] compactions.Compactor(223): Compacting c34ffa13d15949f6980d3190977d1e4a, keycount=2, bloomtype=ROW, size=5.2 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1694032503988 2023-09-06T20:35:15,766 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=83, resume processing ppid=82 2023-09-06T20:35:15,766 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=83, ppid=82, state=SUCCESS; OpenRegionProcedure cc8a9b52139566dcd11c25f6d6fdfaad, server=jenkins-hbase3.apache.org,44489,1694032433466 in 195 msec 2023-09-06T20:35:15,768 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=82, resume processing ppid=81 2023-09-06T20:35:15,768 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=82, ppid=81, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=cc8a9b52139566dcd11c25f6d6fdfaad, ASSIGN in 353 msec 2023-09-06T20:35:15,768 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032515768"}]},"ts":"1694032515768"} 2023-09-06T20:35:15,770 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-09-06T20:35:15,772 INFO [PEWorker-2 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-09-06T20:35:15,774 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=81, state=SUCCESS; EnableTableProcedure table=backup:system in 368 msec 2023-09-06T20:35:15,792 INFO [RS:0;jenkins-hbase3:44489-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc8a9b52139566dcd11c25f6d6fdfaad#session#compaction#17 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:35:15,793 DEBUG [RS:0;jenkins-hbase3:44489-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/6625e06aec104890b76c0277b4ab94f4 is 459, key is session:backup_1694032451196/session:context/1694032468780/Put/seqid=0 2023-09-06T20:35:16,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=81 2023-09-06T20:35:16,012 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(1525): Done restoring backup system table 2023-09-06T20:35:16,012 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:35:16,018 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:35:16,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45185 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:35:16,021 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:35:16,021 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: INCREMENTAL_COPY 2023-09-06T20:35:16,021 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:38135/backupUT/backup_1694032504130/ns2/test-16940324422881 done. 2023-09-06T20:35:16,022 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(208): hdfs://localhost:38135/backupUT/backup_1694032504130/ns2 is empty, remove it. 2023-09-06T20:35:16,023 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:38135/backupUT/backup_1694032504130/default/test-1694032442288 done. 2023-09-06T20:35:16,024 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(208): hdfs://localhost:38135/backupUT/backup_1694032504130/default is empty, remove it. 2023-09-06T20:35:16,026 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:35:16,031 ERROR [Listener at jenkins-hbase3.apache.org/45863 {}] impl.TableBackupClient(246): Backup backup_1694032504130 failed. Backup session finished. Status: FAILURE 2023-09-06T20:35:16,031 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:16,032 ERROR [Listener at jenkins-hbase3.apache.org/45863 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_4 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:168) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_4 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:160) ~[test-classes/:?] ... 41 more 2023-09-06T20:35:16,037 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:16,060 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] hbase.ResourceChecker(175): after: backup.TestIncrementalBackupWithFailures#testIncBackupRestore[0] Thread=601 (was 584) Potentially hanging thread: rs(jenkins-hbase3.apache.org,44489,1694032433466)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0xaacb9cb-shared-pool-32 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataStreamer for file /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/6625e06aec104890b76c0277b4ab94f4 java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:678) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1867771826_16 at /127.0.0.1:45814 [Waiting for operation #25] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,44489,1694032433466)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51523@0x634716f7 java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1455) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1485) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$496/2020798905.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: member: 'jenkins-hbase3.apache.org,44489,1694032433466' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/cluster_17959e2a-7692-c4d5-e8b6-626f5ab10d8e/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1867771826_16 at /127.0.0.1:45118 [Waiting for operation #10] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1867771826_16 at /127.0.0.1:45176 [Waiting for operation #3] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer for 'JobHistoryServer' metrics system java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: rs(jenkins-hbase3.apache.org,44489,1694032433466)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1453139543-172.31.12.81-1694032427287:blk_1073741939_1115, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1867771826_16 at /127.0.0.1:45144 [Waiting for operation #6] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x39685d06-shared-pool-24 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1453139543-172.31.12.81-1694032427287:blk_1073741938_1114, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_762424812_16 at /127.0.0.1:45756 [Receiving block BP-1453139543-172.31.12.81-1694032427287:blk_1073741940_1116] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51523@0x634716f7-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/cluster_17959e2a-7692-c4d5-e8b6-626f5ab10d8e/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: (jenkins-hbase3.apache.org,45185,1694032430454)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:51523@0x634716f7-SendThread(127.0.0.1:51523) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:228) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1338) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1261) Potentially hanging thread: rs(jenkins-hbase3.apache.org,44489,1694032433466)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: region-location-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1453139543-172.31.12.81-1694032427287:blk_1073741940_1116, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS:0;jenkins-hbase3:44489-shortCompactions-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:549) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_762424812_16 at /127.0.0.1:45750 [Receiving block BP-1453139543-172.31.12.81-1694032427287:blk_1073741938_1114] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x39685d06-shared-pool-25 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1867771826_16 at /127.0.0.1:45180 [Waiting for operation #7] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_762424812_16 at /127.0.0.1:45752 [Receiving block BP-1453139543-172.31.12.81-1694032427287:blk_1073741939_1115] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=685 (was 667) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=172 (was 167) - SystemLoadAverage LEAK? -, ProcessCount=167 (was 167), AvailableMemoryMB=3396 (was 3375) - AvailableMemoryMB LEAK? - 2023-09-06T20:35:16,060 WARN [Listener at jenkins-hbase3.apache.org/45863 {}] hbase.ResourceChecker(130): Thread=601 is superior to 500 2023-09-06T20:35:16,070 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-09-06T20:35:16,076 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:16,076 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-09-06T20:35:16,076 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] util.JVMClusterUtil(257): Found active master hash=532317268, stopped=false 2023-09-06T20:35:16,076 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] master.ServerManager(910): Cluster shutdown requested of master=jenkins-hbase3.apache.org,45185,1694032430454 2023-09-06T20:35:16,081 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] procedure2.ProcedureExecutor(630): Stopping 2023-09-06T20:35:16,082 DEBUG [Listener at jenkins-hbase3.apache.org/45863 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:16,082 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,44489,1694032433466' ***** 2023-09-06T20:35:16,083 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] regionserver.HRegionServer(2531): STOPPED: Shutdown requested 2023-09-06T20:35:16,083 INFO [RS:0;jenkins-hbase3:44489 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-09-06T20:35:16,090 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-09-06T20:35:16,092 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-09-06T20:35:16,100 INFO [RS:0;jenkins-hbase3:44489 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@63f72d5b{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-09-06T20:35:16,106 INFO [RS:0;jenkins-hbase3:44489 {}] server.AbstractConnector(383): Stopped ServerConnector@2e45869f{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-09-06T20:35:16,106 INFO [RS:0;jenkins-hbase3:44489 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-09-06T20:35:16,107 INFO [RS:0;jenkins-hbase3:44489 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@1b920999{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-09-06T20:35:16,108 INFO [RS:0;jenkins-hbase3:44489 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@679f1315{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/hadoop.log.dir/,STOPPED} 2023-09-06T20:35:16,111 INFO [RS:0;jenkins-hbase3:44489 {}] regionserver.HeapMemoryManager(220): Stopping 2023-09-06T20:35:16,111 INFO [RS:0;jenkins-hbase3:44489 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2023-09-06T20:35:16,111 INFO [RS:0;jenkins-hbase3:44489 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-09-06T20:35:16,111 INFO [RS:0;jenkins-hbase3:44489 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-09-06T20:35:16,111 INFO [RS:0;jenkins-hbase3:44489 {}] regionserver.HRegionServer(3528): Received CLOSE for 90d54d530e57af59206a5a1415c17690 2023-09-06T20:35:16,112 INFO [RS:0;jenkins-hbase3:44489 {}] regionserver.HRegionServer(3528): Received CLOSE for 2fb717bf2da453848b7afb361b8ed8c0 2023-09-06T20:35:16,112 INFO [RS:0;jenkins-hbase3:44489 {}] regionserver.HRegionServer(3528): Received CLOSE for d8fb0a006da661f8e8078937dc326385 2023-09-06T20:35:16,112 INFO [RS:0;jenkins-hbase3:44489 {}] regionserver.HRegionServer(3528): Received CLOSE for 95242ef863d74dd33bf9cf6336515789 2023-09-06T20:35:16,112 INFO [RS:0;jenkins-hbase3:44489 {}] regionserver.HRegionServer(3528): Received CLOSE for a44b05248e75fb3972b6ec9a34d9f4b8 2023-09-06T20:35:16,112 INFO [RS:0;jenkins-hbase3:44489 {}] regionserver.HRegionServer(3528): Received CLOSE for cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:35:16,112 INFO [RS:0;jenkins-hbase3:44489 {}] regionserver.HRegionServer(3528): Received CLOSE for b0d4786a840d53aeace19d3e7ef71f05 2023-09-06T20:35:16,112 INFO [RS:0;jenkins-hbase3:44489 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,44489,1694032433466 2023-09-06T20:35:16,112 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 90d54d530e57af59206a5a1415c17690, disabling compactions & flushes 2023-09-06T20:35:16,112 DEBUG [RS:0;jenkins-hbase3:44489 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:16,112 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690. 2023-09-06T20:35:16,112 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690. 2023-09-06T20:35:16,112 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690. after waiting 0 ms 2023-09-06T20:35:16,112 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690. 2023-09-06T20:35:16,112 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 90d54d530e57af59206a5a1415c17690 2/2 column families, dataSize=5.30 KB heapSize=12.88 KB 2023-09-06T20:35:16,113 INFO [RS:0;jenkins-hbase3:44489 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-09-06T20:35:16,135 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/90d54d530e57af59206a5a1415c17690/.tmp/f/254f31ad054a499db8e653dee1ca5f2c is 59, key is row-test-1694032442288-1-10/f:q1/1694032468833/Put/seqid=0 2023-09-06T20:35:16,157 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.30 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/90d54d530e57af59206a5a1415c17690/.tmp/f/254f31ad054a499db8e653dee1ca5f2c 2023-09-06T20:35:16,165 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/90d54d530e57af59206a5a1415c17690/.tmp/f/254f31ad054a499db8e653dee1ca5f2c as hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/90d54d530e57af59206a5a1415c17690/f/254f31ad054a499db8e653dee1ca5f2c 2023-09-06T20:35:16,172 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/90d54d530e57af59206a5a1415c17690/f/254f31ad054a499db8e653dee1ca5f2c, entries=99, sequenceid=217, filesize=10.5 K 2023-09-06T20:35:16,173 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~5.30 KB/5425, heapSize ~12.61 KB/12912, currentSize=0 B/0 for 90d54d530e57af59206a5a1415c17690 in 61ms, sequenceid=217, compaction requested=false 2023-09-06T20:35:16,185 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/default/test-1694032442288/90d54d530e57af59206a5a1415c17690/recovered.edits/220.seqid, newMaxSeqId=220, maxSeqId=106 2023-09-06T20:35:16,186 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:35:16,187 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690. 2023-09-06T20:35:16,187 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 90d54d530e57af59206a5a1415c17690: 2023-09-06T20:35:16,187 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1694032442288,,1694032442773.90d54d530e57af59206a5a1415c17690. 2023-09-06T20:35:16,189 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 2fb717bf2da453848b7afb361b8ed8c0, disabling compactions & flushes 2023-09-06T20:35:16,189 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-16940324422883,,1694032448347.2fb717bf2da453848b7afb361b8ed8c0. 2023-09-06T20:35:16,189 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16940324422883,,1694032448347.2fb717bf2da453848b7afb361b8ed8c0. 2023-09-06T20:35:16,189 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16940324422883,,1694032448347.2fb717bf2da453848b7afb361b8ed8c0. after waiting 0 ms 2023-09-06T20:35:16,189 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16940324422883,,1694032448347.2fb717bf2da453848b7afb361b8ed8c0. 2023-09-06T20:35:16,202 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/ns4/test-16940324422883/2fb717bf2da453848b7afb361b8ed8c0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-09-06T20:35:16,207 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:35:16,208 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-16940324422883,,1694032448347.2fb717bf2da453848b7afb361b8ed8c0. 2023-09-06T20:35:16,208 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 2fb717bf2da453848b7afb361b8ed8c0: 2023-09-06T20:35:16,208 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-16940324422883,,1694032448347.2fb717bf2da453848b7afb361b8ed8c0. 2023-09-06T20:35:16,210 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing d8fb0a006da661f8e8078937dc326385, disabling compactions & flushes 2023-09-06T20:35:16,210 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-16940324422882,,1694032446224.d8fb0a006da661f8e8078937dc326385. 2023-09-06T20:35:16,210 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16940324422882,,1694032446224.d8fb0a006da661f8e8078937dc326385. 2023-09-06T20:35:16,210 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16940324422882,,1694032446224.d8fb0a006da661f8e8078937dc326385. after waiting 0 ms 2023-09-06T20:35:16,210 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16940324422882,,1694032446224.d8fb0a006da661f8e8078937dc326385. 2023-09-06T20:35:16,218 DEBUG [RS:0;jenkins-hbase3:44489-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/6625e06aec104890b76c0277b4ab94f4 as hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/6625e06aec104890b76c0277b4ab94f4 2023-09-06T20:35:16,230 DEBUG [RS:0;jenkins-hbase3:44489-longCompactions-0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-09-06T20:35:16,230 INFO [RS:0;jenkins-hbase3:44489-longCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in cc8a9b52139566dcd11c25f6d6fdfaad/session of cc8a9b52139566dcd11c25f6d6fdfaad into 6625e06aec104890b76c0277b4ab94f4(size=6.5 K), total size for store is 6.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:35:16,231 DEBUG [RS:0;jenkins-hbase3:44489-longCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for cc8a9b52139566dcd11c25f6d6fdfaad: 2023-09-06T20:35:16,231 INFO [RS:0;jenkins-hbase3:44489-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad., storeName=cc8a9b52139566dcd11c25f6d6fdfaad/session, priority=13, startTime=1694032515757; duration=0sec 2023-09-06T20:35:16,239 DEBUG [RS:0;jenkins-hbase3:44489-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:35:16,239 DEBUG [RS:0;jenkins-hbase3:44489-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc8a9b52139566dcd11c25f6d6fdfaad:session 2023-09-06T20:35:16,239 INFO [RS:0;jenkins-hbase3:44489 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-09-06T20:35:16,239 INFO [RS:0;jenkins-hbase3:44489 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-09-06T20:35:16,239 INFO [RS:0;jenkins-hbase3:44489 {}] regionserver.HRegionServer(3528): Received CLOSE for 1588230740 2023-09-06T20:35:16,240 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/ns3/test-16940324422882/d8fb0a006da661f8e8078937dc326385/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-09-06T20:35:16,241 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:35:16,241 INFO [RS:0;jenkins-hbase3:44489 {}] regionserver.HRegionServer(1579): Waiting on 6 regions to close 2023-09-06T20:35:16,241 DEBUG [RS:0;jenkins-hbase3:44489 {}] regionserver.HRegionServer(1583): Online Regions={d8fb0a006da661f8e8078937dc326385=ns3:test-16940324422882,,1694032446224.d8fb0a006da661f8e8078937dc326385., 1588230740=hbase:meta,,1.1588230740, 95242ef863d74dd33bf9cf6336515789=ns2:test-16940324422881,,1694032445019.95242ef863d74dd33bf9cf6336515789., a44b05248e75fb3972b6ec9a34d9f4b8=hbase:namespace,,1694032436388.a44b05248e75fb3972b6ec9a34d9f4b8., cc8a9b52139566dcd11c25f6d6fdfaad=backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad., b0d4786a840d53aeace19d3e7ef71f05=backup:system_bulk,,1694032453474.b0d4786a840d53aeace19d3e7ef71f05.} 2023-09-06T20:35:16,249 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-16940324422882,,1694032446224.d8fb0a006da661f8e8078937dc326385. 2023-09-06T20:35:16,249 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for d8fb0a006da661f8e8078937dc326385: 2023-09-06T20:35:16,250 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-16940324422882,,1694032446224.d8fb0a006da661f8e8078937dc326385. 2023-09-06T20:35:16,250 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 95242ef863d74dd33bf9cf6336515789, disabling compactions & flushes 2023-09-06T20:35:16,250 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-16940324422881,,1694032445019.95242ef863d74dd33bf9cf6336515789. 2023-09-06T20:35:16,250 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16940324422881,,1694032445019.95242ef863d74dd33bf9cf6336515789. 2023-09-06T20:35:16,250 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16940324422881,,1694032445019.95242ef863d74dd33bf9cf6336515789. after waiting 0 ms 2023-09-06T20:35:16,250 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16940324422881,,1694032445019.95242ef863d74dd33bf9cf6336515789. 2023-09-06T20:35:16,250 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 95242ef863d74dd33bf9cf6336515789 1/1 column families, dataSize=170 B heapSize=816 B 2023-09-06T20:35:16,251 DEBUG [RS:0;jenkins-hbase3:44489 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 95242ef863d74dd33bf9cf6336515789, a44b05248e75fb3972b6ec9a34d9f4b8, b0d4786a840d53aeace19d3e7ef71f05, cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:35:16,253 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-09-06T20:35:16,253 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-09-06T20:35:16,253 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-09-06T20:35:16,253 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-09-06T20:35:16,253 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-09-06T20:35:16,253 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=18.81 KB heapSize=32.60 KB 2023-09-06T20:35:16,263 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/ns2/test-16940324422881/95242ef863d74dd33bf9cf6336515789/.tmp/f/56f579e02ce94bc99510bb60108d3255 is 38, key is row-t20/f:q1/1694032468944/Put/seqid=0 2023-09-06T20:35:16,276 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/ns2/test-16940324422881/95242ef863d74dd33bf9cf6336515789/.tmp/f/56f579e02ce94bc99510bb60108d3255 2023-09-06T20:35:16,304 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/hbase/meta/1588230740/.tmp/info/adee11dbc9eb4fafab8c9b7e8e949037 is 159, key is ns2:test-16940324422881,,1694032445019.95242ef863d74dd33bf9cf6336515789./info:regioninfo/1694032445805/Put/seqid=0 2023-09-06T20:35:16,316 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/ns2/test-16940324422881/95242ef863d74dd33bf9cf6336515789/.tmp/f/56f579e02ce94bc99510bb60108d3255 as hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/ns2/test-16940324422881/95242ef863d74dd33bf9cf6336515789/f/56f579e02ce94bc99510bb60108d3255 2023-09-06T20:35:16,330 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/ns2/test-16940324422881/95242ef863d74dd33bf9cf6336515789/f/56f579e02ce94bc99510bb60108d3255, entries=5, sequenceid=111, filesize=5.0 K 2023-09-06T20:35:16,331 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~170 B/170, heapSize ~800 B/800, currentSize=0 B/0 for 95242ef863d74dd33bf9cf6336515789 in 81ms, sequenceid=111, compaction requested=false 2023-09-06T20:35:16,346 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/ns2/test-16940324422881/95242ef863d74dd33bf9cf6336515789/recovered.edits/114.seqid, newMaxSeqId=114, maxSeqId=1 2023-09-06T20:35:16,347 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:35:16,348 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-16940324422881,,1694032445019.95242ef863d74dd33bf9cf6336515789. 2023-09-06T20:35:16,348 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 95242ef863d74dd33bf9cf6336515789: 2023-09-06T20:35:16,348 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-16940324422881,,1694032445019.95242ef863d74dd33bf9cf6336515789. 2023-09-06T20:35:16,349 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing a44b05248e75fb3972b6ec9a34d9f4b8, disabling compactions & flushes 2023-09-06T20:35:16,349 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1694032436388.a44b05248e75fb3972b6ec9a34d9f4b8. 2023-09-06T20:35:16,349 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1694032436388.a44b05248e75fb3972b6ec9a34d9f4b8. 2023-09-06T20:35:16,349 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1694032436388.a44b05248e75fb3972b6ec9a34d9f4b8. after waiting 0 ms 2023-09-06T20:35:16,349 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1694032436388.a44b05248e75fb3972b6ec9a34d9f4b8. 2023-09-06T20:35:16,349 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing a44b05248e75fb3972b6ec9a34d9f4b8 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-09-06T20:35:16,368 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/hbase/namespace/a44b05248e75fb3972b6ec9a34d9f4b8/.tmp/info/b752e030590446fa96237f282481d68c is 45, key is default/info:d/1694032437763/Put/seqid=0 2023-09-06T20:35:16,376 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/hbase/namespace/a44b05248e75fb3972b6ec9a34d9f4b8/.tmp/info/b752e030590446fa96237f282481d68c 2023-09-06T20:35:16,392 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/hbase/namespace/a44b05248e75fb3972b6ec9a34d9f4b8/.tmp/info/b752e030590446fa96237f282481d68c as hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/hbase/namespace/a44b05248e75fb3972b6ec9a34d9f4b8/info/b752e030590446fa96237f282481d68c 2023-09-06T20:35:16,398 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/hbase/namespace/a44b05248e75fb3972b6ec9a34d9f4b8/info/b752e030590446fa96237f282481d68c, entries=7, sequenceid=11, filesize=5.1 K 2023-09-06T20:35:16,400 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for a44b05248e75fb3972b6ec9a34d9f4b8 in 51ms, sequenceid=11, compaction requested=false 2023-09-06T20:35:16,414 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/hbase/namespace/a44b05248e75fb3972b6ec9a34d9f4b8/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-09-06T20:35:16,415 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:35:16,416 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1694032436388.a44b05248e75fb3972b6ec9a34d9f4b8. 2023-09-06T20:35:16,416 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for a44b05248e75fb3972b6ec9a34d9f4b8: 2023-09-06T20:35:16,416 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1694032436388.a44b05248e75fb3972b6ec9a34d9f4b8. 2023-09-06T20:35:16,416 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing cc8a9b52139566dcd11c25f6d6fdfaad, disabling compactions & flushes 2023-09-06T20:35:16,416 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:35:16,416 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:35:16,416 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. after waiting 0 ms 2023-09-06T20:35:16,416 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:35:16,416 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing cc8a9b52139566dcd11c25f6d6fdfaad 2/2 column families, dataSize=296 B heapSize=960 B 2023-09-06T20:35:16,420 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/690f40a373914af8be4c0bc9ea43ea99 is 256, key is session:backup_1694032504130/session:context/1694032516025/Put/seqid=0 2023-09-06T20:35:16,437 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-09-06T20:35:16,437 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-09-06T20:35:16,452 DEBUG [RS:0;jenkins-hbase3:44489 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, b0d4786a840d53aeace19d3e7ef71f05, cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:35:16,652 DEBUG [RS:0;jenkins-hbase3:44489 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, b0d4786a840d53aeace19d3e7ef71f05, cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:35:16,740 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.41 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/hbase/meta/1588230740/.tmp/info/adee11dbc9eb4fafab8c9b7e8e949037 2023-09-06T20:35:16,758 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/hbase/meta/1588230740/.tmp/table/932feaeb33bb4649a8e3f547f0ff37c4 is 59, key is ns2:test-16940324422881/table:state/1694032445814/Put/seqid=0 2023-09-06T20:35:16,764 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.40 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/hbase/meta/1588230740/.tmp/table/932feaeb33bb4649a8e3f547f0ff37c4 2023-09-06T20:35:16,770 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/hbase/meta/1588230740/.tmp/info/adee11dbc9eb4fafab8c9b7e8e949037 as hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/hbase/meta/1588230740/info/adee11dbc9eb4fafab8c9b7e8e949037 2023-09-06T20:35:16,774 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/hbase/meta/1588230740/info/adee11dbc9eb4fafab8c9b7e8e949037, entries=79, sequenceid=83, filesize=15.1 K 2023-09-06T20:35:16,775 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/hbase/meta/1588230740/.tmp/table/932feaeb33bb4649a8e3f547f0ff37c4 as hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/hbase/meta/1588230740/table/932feaeb33bb4649a8e3f547f0ff37c4 2023-09-06T20:35:16,780 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/hbase/meta/1588230740/table/932feaeb33bb4649a8e3f547f0ff37c4, entries=15, sequenceid=83, filesize=5.8 K 2023-09-06T20:35:16,781 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~18.81 KB/19263, heapSize ~32.32 KB/33096, currentSize=0 B/0 for 1588230740 in 528ms, sequenceid=83, compaction requested=false 2023-09-06T20:35:16,793 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/hbase/meta/1588230740/recovered.edits/86.seqid, newMaxSeqId=86, maxSeqId=1 2023-09-06T20:35:16,794 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:35:16,794 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-09-06T20:35:16,794 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-09-06T20:35:16,794 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-09-06T20:35:16,794 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-09-06T20:35:16,830 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=296 B at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/690f40a373914af8be4c0bc9ea43ea99 2023-09-06T20:35:16,836 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/.tmp/session/690f40a373914af8be4c0bc9ea43ea99 as hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/690f40a373914af8be4c0bc9ea43ea99 2023-09-06T20:35:16,841 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/690f40a373914af8be4c0bc9ea43ea99, entries=2, sequenceid=82, filesize=5.2 K 2023-09-06T20:35:16,842 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~296 B/296, heapSize ~688 B/688, currentSize=0 B/0 for cc8a9b52139566dcd11c25f6d6fdfaad in 426ms, sequenceid=82, compaction requested=false 2023-09-06T20:35:16,850 DEBUG [StoreCloser-backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/22fa315304564a1397a14e4ba5bc21c8, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/7772e4890a534e54a647e460587efe32, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/c34ffa13d15949f6980d3190977d1e4a] to archive 2023-09-06T20:35:16,851 DEBUG [StoreCloser-backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-06T20:35:16,852 DEBUG [RS:0;jenkins-hbase3:44489 {}] regionserver.HRegionServer(1609): Waiting on b0d4786a840d53aeace19d3e7ef71f05, cc8a9b52139566dcd11c25f6d6fdfaad 2023-09-06T20:35:16,853 DEBUG [StoreCloser-backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/22fa315304564a1397a14e4ba5bc21c8 to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/archive/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/22fa315304564a1397a14e4ba5bc21c8 2023-09-06T20:35:16,855 DEBUG [StoreCloser-backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/7772e4890a534e54a647e460587efe32 to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/archive/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/7772e4890a534e54a647e460587efe32 2023-09-06T20:35:16,857 DEBUG [StoreCloser-backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/c34ffa13d15949f6980d3190977d1e4a to hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/archive/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/session/c34ffa13d15949f6980d3190977d1e4a 2023-09-06T20:35:16,864 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system/cc8a9b52139566dcd11c25f6d6fdfaad/recovered.edits/85.seqid, newMaxSeqId=85, maxSeqId=76 2023-09-06T20:35:16,865 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:35:16,865 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:35:16,865 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for cc8a9b52139566dcd11c25f6d6fdfaad: 2023-09-06T20:35:16,865 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1694032451346.cc8a9b52139566dcd11c25f6d6fdfaad. 2023-09-06T20:35:16,866 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing b0d4786a840d53aeace19d3e7ef71f05, disabling compactions & flushes 2023-09-06T20:35:16,866 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1694032453474.b0d4786a840d53aeace19d3e7ef71f05. 2023-09-06T20:35:16,866 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1694032453474.b0d4786a840d53aeace19d3e7ef71f05. 2023-09-06T20:35:16,866 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1694032453474.b0d4786a840d53aeace19d3e7ef71f05. after waiting 0 ms 2023-09-06T20:35:16,866 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1694032453474.b0d4786a840d53aeace19d3e7ef71f05. 2023-09-06T20:35:16,868 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/data/backup/system_bulk/b0d4786a840d53aeace19d3e7ef71f05/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-09-06T20:35:16,869 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:35:16,869 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1694032453474.b0d4786a840d53aeace19d3e7ef71f05. 2023-09-06T20:35:16,869 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for b0d4786a840d53aeace19d3e7ef71f05: 2023-09-06T20:35:16,869 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1694032453474.b0d4786a840d53aeace19d3e7ef71f05. 2023-09-06T20:35:17,052 INFO [RS:0;jenkins-hbase3:44489 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,44489,1694032433466; all regions closed. 2023-09-06T20:35:17,063 DEBUG [RS:0;jenkins-hbase3:44489 {}] wal.AbstractFSWAL(1062): Moved 5 WAL file(s) to /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/oldWALs 2023-09-06T20:35:17,063 INFO [RS:0;jenkins-hbase3:44489 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C44489%2C1694032433466.meta:.meta(num 1694032505328) 2023-09-06T20:35:17,070 DEBUG [RS:0;jenkins-hbase3:44489 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/oldWALs 2023-09-06T20:35:17,070 INFO [RS:0;jenkins-hbase3:44489 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-0:(num 1694032505293) 2023-09-06T20:35:17,082 DEBUG [RS:0;jenkins-hbase3:44489 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/oldWALs 2023-09-06T20:35:17,082 INFO [RS:0;jenkins-hbase3:44489 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C44489%2C1694032433466.jenkins-hbase3.apache.org%2C44489%2C1694032433466.regiongroup-1:(num 1694032505312) 2023-09-06T20:35:17,082 DEBUG [RS:0;jenkins-hbase3:44489 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:17,082 INFO [RS:0;jenkins-hbase3:44489 {}] regionserver.LeaseManager(133): Closed leases 2023-09-06T20:35:17,083 INFO [RS:0;jenkins-hbase3:44489 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2023-09-06T20:35:17,083 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-09-06T20:35:17,085 INFO [RS:0;jenkins-hbase3:44489 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:44489 2023-09-06T20:35:17,097 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,44489,1694032433466] 2023-09-06T20:35:17,097 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,44489,1694032433466; numProcessing=1 2023-09-06T20:35:17,098 INFO [RegionServerTracker-0 {}] master.ServerManager(569): Cluster shutdown set; jenkins-hbase3.apache.org,44489,1694032433466 expired; onlineServers=0 2023-09-06T20:35:17,098 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,45185,1694032430454' ***** 2023-09-06T20:35:17,098 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2531): STOPPED: Cluster shutdown set; onlineServer=0 2023-09-06T20:35:17,099 DEBUG [M:0;jenkins-hbase3:45185 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57763113, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-09-06T20:35:17,099 INFO [M:0;jenkins-hbase3:45185 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-09-06T20:35:17,156 INFO [M:0;jenkins-hbase3:45185 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@5d1c5812{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-09-06T20:35:17,157 INFO [M:0;jenkins-hbase3:45185 {}] server.AbstractConnector(383): Stopped ServerConnector@452071dd{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-09-06T20:35:17,157 INFO [M:0;jenkins-hbase3:45185 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-09-06T20:35:17,158 INFO [M:0;jenkins-hbase3:45185 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@2c6b8bd5{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-09-06T20:35:17,159 INFO [M:0;jenkins-hbase3:45185 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@55869636{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/hadoop.log.dir/,STOPPED} 2023-09-06T20:35:17,159 INFO [M:0;jenkins-hbase3:45185 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,45185,1694032430454 2023-09-06T20:35:17,159 INFO [M:0;jenkins-hbase3:45185 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,45185,1694032430454; all regions closed. 2023-09-06T20:35:17,159 DEBUG [M:0;jenkins-hbase3:45185 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:17,159 INFO [M:0;jenkins-hbase3:45185 {}] master.HMaster(1665): Stopping master jetty server 2023-09-06T20:35:17,160 INFO [M:0;jenkins-hbase3:45185 {}] server.AbstractConnector(383): Stopped ServerConnector@302e9b29{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-09-06T20:35:17,160 DEBUG [M:0;jenkins-hbase3:45185 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-09-06T20:35:17,160 INFO [M:0;jenkins-hbase3:45185 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-09-06T20:35:17,161 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-09-06T20:35:17,161 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1694032435232 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1694032435232,5,FailOnTimeoutGroup] 2023-09-06T20:35:17,161 DEBUG [M:0;jenkins-hbase3:45185 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:35:17,161 DEBUG [M:0;jenkins-hbase3:45185 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-09-06T20:35:17,161 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1694032435234 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1694032435234,5,FailOnTimeoutGroup] 2023-09-06T20:35:17,161 INFO [M:0;jenkins-hbase3:45185 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-09-06T20:35:17,161 DEBUG [M:0;jenkins-hbase3:45185 {}] master.HMaster(1688): Stopping service threads 2023-09-06T20:35:17,161 INFO [M:0;jenkins-hbase3:45185 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-09-06T20:35:17,161 ERROR [M:0;jenkins-hbase3:45185 {}] procedure2.ProcedureExecutor(654): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] 2023-09-06T20:35:17,162 INFO [M:0;jenkins-hbase3:45185 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-09-06T20:35:17,162 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-09-06T20:35:17,163 WARN [M:0;jenkins-hbase3:45185 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-09-06T20:35:17,163 INFO [M:0;jenkins-hbase3:45185 {}] assignment.AssignmentManager(382): Stopping assignment manager 2023-09-06T20:35:17,163 INFO [M:0;jenkins-hbase3:45185 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-09-06T20:35:17,164 DEBUG [M:0;jenkins-hbase3:45185 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-09-06T20:35:17,178 INFO [M:0;jenkins-hbase3:45185 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:35:17,179 DEBUG [M:0;jenkins-hbase3:45185 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:35:17,179 DEBUG [M:0;jenkins-hbase3:45185 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-09-06T20:35:17,179 DEBUG [M:0;jenkins-hbase3:45185 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:35:17,179 INFO [M:0;jenkins-hbase3:45185 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=325.90 KB heapSize=389.66 KB 2023-09-06T20:35:17,192 DEBUG [M:0;jenkins-hbase3:45185 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/033b64da6e68439db9dd961aa86e7ad1 is 88, key is hbase:meta,,1/info:sn/1694032435702/Put/seqid=0 2023-09-06T20:35:17,197 INFO [RS:0;jenkins-hbase3:44489 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,44489,1694032433466; zookeeper connection closed. 2023-09-06T20:35:17,197 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@b47f647 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@b47f647 2023-09-06T20:35:17,197 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-09-06T20:35:17,598 INFO [M:0;jenkins-hbase3:45185 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=758 (bloomFilter=true), to=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/033b64da6e68439db9dd961aa86e7ad1 2023-09-06T20:35:17,621 DEBUG [M:0;jenkins-hbase3:45185 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/45d228a7101848fdb5ec95d12d1a405c is 1281, key is \x00\x00\x00\x00\x00\x00\x00\x19/proc:d/1694032451158/Put/seqid=0 2023-09-06T20:35:17,628 INFO [M:0;jenkins-hbase3:45185 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=325.31 KB at sequenceid=758 (bloomFilter=true), to=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/45d228a7101848fdb5ec95d12d1a405c 2023-09-06T20:35:17,634 INFO [M:0;jenkins-hbase3:45185 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 45d228a7101848fdb5ec95d12d1a405c 2023-09-06T20:35:17,650 DEBUG [M:0;jenkins-hbase3:45185 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/843a6c85936644d298b12ca754e568a1 is 82, key is jenkins-hbase3.apache.org,44489,1694032433466/rs:state/1694032435353/Put/seqid=0 2023-09-06T20:35:18,060 INFO [M:0;jenkins-hbase3:45185 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=758 (bloomFilter=true), to=hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/843a6c85936644d298b12ca754e568a1 2023-09-06T20:35:18,067 DEBUG [M:0;jenkins-hbase3:45185 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/033b64da6e68439db9dd961aa86e7ad1 as hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/033b64da6e68439db9dd961aa86e7ad1 2023-09-06T20:35:18,073 INFO [M:0;jenkins-hbase3:45185 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/033b64da6e68439db9dd961aa86e7ad1, entries=8, sequenceid=758, filesize=5.5 K 2023-09-06T20:35:18,074 DEBUG [M:0;jenkins-hbase3:45185 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/45d228a7101848fdb5ec95d12d1a405c as hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/45d228a7101848fdb5ec95d12d1a405c 2023-09-06T20:35:18,081 INFO [M:0;jenkins-hbase3:45185 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 45d228a7101848fdb5ec95d12d1a405c 2023-09-06T20:35:18,082 INFO [M:0;jenkins-hbase3:45185 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/45d228a7101848fdb5ec95d12d1a405c, entries=83, sequenceid=758, filesize=26.0 K 2023-09-06T20:35:18,083 DEBUG [M:0;jenkins-hbase3:45185 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/843a6c85936644d298b12ca754e568a1 as hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/843a6c85936644d298b12ca754e568a1 2023-09-06T20:35:18,091 INFO [M:0;jenkins-hbase3:45185 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38135/user/jenkins/test-data/af8a035c-ce46-847a-4924-5df75f485928/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/843a6c85936644d298b12ca754e568a1, entries=1, sequenceid=758, filesize=5.1 K 2023-09-06T20:35:18,092 INFO [M:0;jenkins-hbase3:45185 {}] regionserver.HRegion(3022): Finished flush of dataSize ~325.90 KB/333726, heapSize ~389.36 KB/398704, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 913ms, sequenceid=758, compaction requested=false 2023-09-06T20:35:18,097 INFO [M:0;jenkins-hbase3:45185 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:35:18,097 DEBUG [M:0;jenkins-hbase3:45185 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-09-06T20:35:18,101 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-09-06T20:35:18,101 INFO [M:0;jenkins-hbase3:45185 {}] flush.MasterFlushTableProcedureManager(85): stop: server shutting down. 2023-09-06T20:35:18,101 INFO [M:0;jenkins-hbase3:45185 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-09-06T20:35:18,102 INFO [M:0;jenkins-hbase3:45185 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:45185 2023-09-06T20:35:18,129 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:35:18,206 INFO [M:0;jenkins-hbase3:45185 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,45185,1694032430454; zookeeper connection closed. 2023-09-06T20:35:18,212 WARN [Listener at jenkins-hbase3.apache.org/45863 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-09-06T20:35:18,216 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-09-06T20:35:18,324 WARN [BP-1453139543-172.31.12.81-1694032427287 heartbeating to localhost/127.0.0.1:38135 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-09-06T20:35:18,324 WARN [BP-1453139543-172.31.12.81-1694032427287 heartbeating to localhost/127.0.0.1:38135 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-1453139543-172.31.12.81-1694032427287 (Datanode Uuid cedb6886-cf3f-4a99-b36b-a59b3394261c) service to localhost/127.0.0.1:38135 2023-09-06T20:35:18,326 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/cluster_17959e2a-7692-c4d5-e8b6-626f5ab10d8e/dfs/data/data1/current/BP-1453139543-172.31.12.81-1694032427287 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-09-06T20:35:18,326 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b0d673e3-772d-6aac-8b70-460ea0f0cc6e/cluster_17959e2a-7692-c4d5-e8b6-626f5ab10d8e/dfs/data/data2/current/BP-1453139543-172.31.12.81-1694032427287 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-09-06T20:35:18,351 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-09-06T20:35:18,529 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-09-06T20:35:18,529 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-09-06T20:35:18,538 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-09-06T20:35:18,653 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-09-06T20:35:18,764 ERROR [Thread[Thread-277,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-09-06T20:35:18,765 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-09-06T20:35:18,869 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-09-06T20:35:18,879 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-09-06T20:35:18,881 ERROR [Thread[Thread-287,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-09-06T20:35:18,887 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-09-06T20:35:18,989 ERROR [Thread[Thread-252,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-09-06T20:35:18,989 INFO [Listener at jenkins-hbase3.apache.org/45863 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped