2023-09-06 20:38:27,915 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 2023-09-06 20:38:27,928 main DEBUG Took 0.010494 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-09-06 20:38:27,928 main DEBUG PluginManager 'Core' found 129 plugins 2023-09-06 20:38:27,928 main DEBUG PluginManager 'Level' found 0 plugins 2023-09-06 20:38:27,929 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-09-06 20:38:27,930 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:38:27,938 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-09-06 20:38:27,950 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:38:27,952 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:38:27,952 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:38:27,952 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:38:27,953 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:38:27,953 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:38:27,954 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:38:27,954 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:38:27,954 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:38:27,954 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:38:27,955 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:38:27,955 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:38:27,956 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:38:27,956 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:38:27,956 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:38:27,956 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:38:27,957 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:38:27,957 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:38:27,957 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:38:27,958 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:38:27,958 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:38:27,958 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:38:27,959 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:38:27,959 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-09-06 20:38:27,959 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:38:27,959 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-09-06 20:38:27,961 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-09-06 20:38:27,963 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-09-06 20:38:27,964 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-09-06 20:38:27,965 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-09-06 20:38:27,966 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-09-06 20:38:27,966 main DEBUG PluginManager 'Converter' found 47 plugins 2023-09-06 20:38:27,979 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-09-06 20:38:27,981 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-09-06 20:38:27,985 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-09-06 20:38:27,985 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-09-06 20:38:27,986 main DEBUG createAppenders(={Console}) 2023-09-06 20:38:27,986 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 initialized 2023-09-06 20:38:27,987 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 2023-09-06 20:38:27,987 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 OK. 2023-09-06 20:38:27,987 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-09-06 20:38:27,988 main DEBUG OutputStream closed 2023-09-06 20:38:27,988 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-09-06 20:38:27,988 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-09-06 20:38:27,988 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@2145b572 OK 2023-09-06 20:38:28,031 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-09-06 20:38:28,033 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-09-06 20:38:28,034 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-09-06 20:38:28,035 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-09-06 20:38:28,035 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-09-06 20:38:28,036 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-09-06 20:38:28,036 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-09-06 20:38:28,036 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-09-06 20:38:28,036 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-09-06 20:38:28,037 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-09-06 20:38:28,037 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-09-06 20:38:28,037 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-09-06 20:38:28,038 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-09-06 20:38:28,038 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-09-06 20:38:28,038 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-09-06 20:38:28,038 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-09-06 20:38:28,039 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-09-06 20:38:28,040 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-09-06 20:38:28,042 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-09-06 20:38:28,043 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@49912c99) with optional ClassLoader: null 2023-09-06 20:38:28,043 main DEBUG Shutdown hook enabled. Registering a new one. 2023-09-06 20:38:28,044 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@49912c99] started OK. 2023-09-06T20:38:28,074 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestFullBackupWithFailures timeout: 13 mins 2023-09-06 20:38:28,078 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-09-06 20:38:28,078 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-09-06T20:38:28,418 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2 2023-09-06T20:38:28,422 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-09-06T20:38:28,425 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:38:28,426 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-09-06T20:38:28,429 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/cluster_b47b59a8-e7b8-2208-bc7e-15cb99b544fd, deleteOnExit=true 2023-09-06T20:38:28,430 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-09-06T20:38:28,430 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/test.cache.data in system properties and HBase conf 2023-09-06T20:38:28,431 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/hadoop.tmp.dir in system properties and HBase conf 2023-09-06T20:38:28,431 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/hadoop.log.dir in system properties and HBase conf 2023-09-06T20:38:28,432 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/mapreduce.cluster.local.dir in system properties and HBase conf 2023-09-06T20:38:28,432 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-09-06T20:38:28,432 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-09-06T20:38:28,534 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-09-06T20:38:28,988 DEBUG [Time-limited test {}] fs.HFileSystem(308): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-09-06T20:38:28,993 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-09-06T20:38:28,994 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-09-06T20:38:28,994 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-09-06T20:38:28,994 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-06T20:38:28,995 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-09-06T20:38:28,995 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-09-06T20:38:28,995 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-06T20:38:28,996 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-06T20:38:28,996 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-09-06T20:38:28,996 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/nfs.dump.dir in system properties and HBase conf 2023-09-06T20:38:28,996 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/java.io.tmpdir in system properties and HBase conf 2023-09-06T20:38:28,997 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-06T20:38:28,997 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-09-06T20:38:28,997 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-09-06T20:38:29,529 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-09-06T20:38:29,534 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-09-06T20:38:29,821 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-09-06T20:38:30,016 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@6b8729fb via org.mortbay.log.Slf4jLog 2023-09-06T20:38:30,037 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:38:30,076 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:38:30,121 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/java.io.tmpdir/Jetty_localhost_35443_hdfs____.jemwcl/webapp 2023-09-06T20:38:30,280 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:35443 2023-09-06T20:38:30,300 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-09-06T20:38:30,300 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-09-06T20:38:30,949 WARN [Listener at localhost/34451 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-09-06T20:38:31,027 WARN [Listener at localhost/34451 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-09-06T20:38:31,048 WARN [Listener at localhost/34451 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:38:31,056 INFO [Listener at localhost/34451 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:38:31,065 INFO [Listener at localhost/34451 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/java.io.tmpdir/Jetty_localhost_39287_datanode____5u8vsi/webapp 2023-09-06T20:38:31,180 INFO [Listener at localhost/34451 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:39287 2023-09-06T20:38:31,538 WARN [Listener at localhost/38777 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-09-06T20:38:31,872 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-09-06T20:38:32,071 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xde61391a2b13f050: Processing first storage report for DS-07aa54ab-1cb2-4831-8bde-a715b49e632a from datanode 8df01a43-0f69-4a0f-85c1-859f5e9f0945 2023-09-06T20:38:32,073 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xde61391a2b13f050: from storage DS-07aa54ab-1cb2-4831-8bde-a715b49e632a node DatanodeRegistration(127.0.0.1:41237, datanodeUuid=8df01a43-0f69-4a0f-85c1-859f5e9f0945, infoPort=33171, infoSecurePort=0, ipcPort=38777, storageInfo=lv=-57;cid=testClusterID;nsid=251621914;c=1694032709610), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2023-09-06T20:38:32,074 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xde61391a2b13f050: Processing first storage report for DS-b3fc0486-d96a-43eb-b38f-a4aa3949688e from datanode 8df01a43-0f69-4a0f-85c1-859f5e9f0945 2023-09-06T20:38:32,074 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xde61391a2b13f050: from storage DS-b3fc0486-d96a-43eb-b38f-a4aa3949688e node DatanodeRegistration(127.0.0.1:41237, datanodeUuid=8df01a43-0f69-4a0f-85c1-859f5e9f0945, infoPort=33171, infoSecurePort=0, ipcPort=38777, storageInfo=lv=-57;cid=testClusterID;nsid=251621914;c=1694032709610), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-09-06T20:38:32,125 DEBUG [Listener at localhost/38777 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2 2023-09-06T20:38:32,213 INFO [Listener at localhost/38777 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/cluster_b47b59a8-e7b8-2208-bc7e-15cb99b544fd/zookeeper_0, clientPort=58035, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/cluster_b47b59a8-e7b8-2208-bc7e-15cb99b544fd/zookeeper_0/version-2, dataDirSize=424 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/cluster_b47b59a8-e7b8-2208-bc7e-15cb99b544fd/zookeeper_0/version-2, dataLogSize=424 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, serverId=0 2023-09-06T20:38:32,240 INFO [Listener at localhost/38777 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=58035 2023-09-06T20:38:32,250 INFO [Listener at localhost/38777 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:38:32,252 INFO [Listener at localhost/38777 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:38:32,946 INFO [Listener at localhost/38777 {}] util.FSUtils(461): Created version file at hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0 with version=8 2023-09-06T20:38:32,946 INFO [Listener at localhost/38777 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/hbase-staging 2023-09-06T20:38:32,963 DEBUG [Listener at localhost/38777 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-09-06T20:38:32,963 DEBUG [Listener at localhost/38777 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-09-06T20:38:32,963 DEBUG [Listener at localhost/38777 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-09-06T20:38:32,963 DEBUG [Listener at localhost/38777 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-09-06T20:38:33,184 DEBUG [Listener at localhost/38777 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-09-06T20:38:33,941 INFO [Listener at localhost/38777 {}] client.ConnectionUtils(128): master/jenkins-hbase3:0 server-side Connection retries=45 2023-09-06T20:38:33,988 INFO [Listener at localhost/38777 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-06T20:38:33,989 INFO [Listener at localhost/38777 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-09-06T20:38:33,989 INFO [Listener at localhost/38777 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-09-06T20:38:33,989 INFO [Listener at localhost/38777 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-06T20:38:33,989 INFO [Listener at localhost/38777 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-09-06T20:38:34,141 INFO [Listener at localhost/38777 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-09-06T20:38:34,276 INFO [Listener at localhost/38777 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-09-06T20:38:34,295 DEBUG [Listener at localhost/38777 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-09-06T20:38:34,302 INFO [Listener at localhost/38777 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-09-06T20:38:34,352 DEBUG [Listener at localhost/38777 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 17520 (auto-detected) 2023-09-06T20:38:34,354 DEBUG [Listener at localhost/38777 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:1c:ff:fe:e2:13:ea (auto-detected) 2023-09-06T20:38:34,392 INFO [Listener at localhost/38777 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:41705 2023-09-06T20:38:34,407 INFO [Listener at localhost/38777 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:38:34,409 INFO [Listener at localhost/38777 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:38:34,430 INFO [Listener at localhost/38777 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:41705 connecting to ZooKeeper ensemble=127.0.0.1:58035 2023-09-06T20:38:34,478 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): master:417050x0, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-09-06T20:38:34,483 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:41705-0x100ef01325b0000 connected 2023-09-06T20:38:34,507 DEBUG [Listener at localhost/38777 {}] zookeeper.ZKUtil(113): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-09-06T20:38:34,510 DEBUG [Listener at localhost/38777 {}] zookeeper.ZKUtil(113): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-09-06T20:38:34,514 DEBUG [Listener at localhost/38777 {}] zookeeper.ZKUtil(113): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-09-06T20:38:34,525 DEBUG [Listener at localhost/38777 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41705 2023-09-06T20:38:34,525 DEBUG [Listener at localhost/38777 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41705 2023-09-06T20:38:34,526 DEBUG [Listener at localhost/38777 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41705 2023-09-06T20:38:34,526 DEBUG [Listener at localhost/38777 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41705 2023-09-06T20:38:34,526 DEBUG [Listener at localhost/38777 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41705 2023-09-06T20:38:34,580 INFO [Listener at localhost/38777 {}] log.Log(170): Logging initialized @7392ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-09-06T20:38:34,704 INFO [Listener at localhost/38777 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-09-06T20:38:34,705 INFO [Listener at localhost/38777 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-09-06T20:38:34,706 INFO [Listener at localhost/38777 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-09-06T20:38:34,708 INFO [Listener at localhost/38777 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-09-06T20:38:34,708 INFO [Listener at localhost/38777 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-09-06T20:38:34,708 INFO [Listener at localhost/38777 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-09-06T20:38:34,711 INFO [Listener at localhost/38777 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-09-06T20:38:34,842 INFO [Listener at localhost/38777 {}] http.HttpServer(1219): Jetty bound to port 44233 2023-09-06T20:38:34,844 INFO [Listener at localhost/38777 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-09-06T20:38:34,874 INFO [Listener at localhost/38777 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:38:34,878 INFO [Listener at localhost/38777 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@e213d8e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/hadoop.log.dir/,AVAILABLE} 2023-09-06T20:38:34,878 INFO [Listener at localhost/38777 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:38:34,879 INFO [Listener at localhost/38777 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@5bca7cba{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-09-06T20:38:34,942 INFO [Listener at localhost/38777 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-09-06T20:38:34,955 INFO [Listener at localhost/38777 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-09-06T20:38:34,955 INFO [Listener at localhost/38777 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-09-06T20:38:34,957 INFO [Listener at localhost/38777 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2023-09-06T20:38:34,964 INFO [Listener at localhost/38777 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:38:36,097 INFO [Listener at localhost/38777 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@53dd1486{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-09-06T20:38:36,109 INFO [Listener at localhost/38777 {}] server.AbstractConnector(333): Started ServerConnector@19e02c9c{HTTP/1.1, (http/1.1)}{0.0.0.0:44233} 2023-09-06T20:38:36,110 INFO [Listener at localhost/38777 {}] server.Server(415): Started @8922ms 2023-09-06T20:38:36,113 INFO [Listener at localhost/38777 {}] master.HMaster(485): hbase.rootdir=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0, hbase.cluster.distributed=false 2023-09-06T20:38:36,173 INFO [Listener at localhost/38777 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase3:0 server-side Connection retries=45 2023-09-06T20:38:36,174 INFO [Listener at localhost/38777 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-06T20:38:36,174 INFO [Listener at localhost/38777 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-09-06T20:38:36,174 INFO [Listener at localhost/38777 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-09-06T20:38:36,175 INFO [Listener at localhost/38777 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-09-06T20:38:36,175 INFO [Listener at localhost/38777 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-09-06T20:38:36,186 INFO [Listener at localhost/38777 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-09-06T20:38:36,189 INFO [Listener at localhost/38777 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-09-06T20:38:36,190 INFO [Listener at localhost/38777 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:45559 2023-09-06T20:38:36,193 INFO [Listener at localhost/38777 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-09-06T20:38:36,201 DEBUG [Listener at localhost/38777 {}] mob.MobFileCache(120): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-09-06T20:38:36,201 INFO [Listener at localhost/38777 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:38:36,204 INFO [Listener at localhost/38777 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:38:36,206 INFO [Listener at localhost/38777 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:45559 connecting to ZooKeeper ensemble=127.0.0.1:58035 2023-09-06T20:38:36,209 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): regionserver:455590x0, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-09-06T20:38:36,210 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:45559-0x100ef01325b0001 connected 2023-09-06T20:38:36,210 DEBUG [Listener at localhost/38777 {}] zookeeper.ZKUtil(113): regionserver:45559-0x100ef01325b0001, quorum=127.0.0.1:58035, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-09-06T20:38:36,212 DEBUG [Listener at localhost/38777 {}] zookeeper.ZKUtil(113): regionserver:45559-0x100ef01325b0001, quorum=127.0.0.1:58035, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-09-06T20:38:36,212 DEBUG [Listener at localhost/38777 {}] zookeeper.ZKUtil(113): regionserver:45559-0x100ef01325b0001, quorum=127.0.0.1:58035, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-09-06T20:38:36,213 DEBUG [Listener at localhost/38777 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45559 2023-09-06T20:38:36,213 DEBUG [Listener at localhost/38777 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45559 2023-09-06T20:38:36,213 DEBUG [Listener at localhost/38777 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45559 2023-09-06T20:38:36,214 DEBUG [Listener at localhost/38777 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45559 2023-09-06T20:38:36,214 DEBUG [Listener at localhost/38777 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45559 2023-09-06T20:38:36,218 INFO [Listener at localhost/38777 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-09-06T20:38:36,218 INFO [Listener at localhost/38777 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-09-06T20:38:36,219 INFO [Listener at localhost/38777 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-09-06T20:38:36,219 INFO [Listener at localhost/38777 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-09-06T20:38:36,220 INFO [Listener at localhost/38777 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-09-06T20:38:36,220 INFO [Listener at localhost/38777 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-09-06T20:38:36,220 INFO [Listener at localhost/38777 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-09-06T20:38:36,223 INFO [Listener at localhost/38777 {}] http.HttpServer(1219): Jetty bound to port 33473 2023-09-06T20:38:36,223 INFO [Listener at localhost/38777 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-09-06T20:38:36,224 INFO [Listener at localhost/38777 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:38:36,224 INFO [Listener at localhost/38777 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@740cdc54{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/hadoop.log.dir/,AVAILABLE} 2023-09-06T20:38:36,225 INFO [Listener at localhost/38777 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:38:36,225 INFO [Listener at localhost/38777 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@5fdc8a81{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-09-06T20:38:36,236 INFO [Listener at localhost/38777 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-09-06T20:38:36,237 INFO [Listener at localhost/38777 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-09-06T20:38:36,238 INFO [Listener at localhost/38777 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-09-06T20:38:36,238 INFO [Listener at localhost/38777 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2023-09-06T20:38:36,239 INFO [Listener at localhost/38777 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-09-06T20:38:36,243 INFO [Listener at localhost/38777 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@6b0a0cdf{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-09-06T20:38:36,244 INFO [Listener at localhost/38777 {}] server.AbstractConnector(333): Started ServerConnector@2ef3cb42{HTTP/1.1, (http/1.1)}{0.0.0.0:33473} 2023-09-06T20:38:36,244 INFO [Listener at localhost/38777 {}] server.Server(415): Started @9057ms 2023-09-06T20:38:36,249 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-09-06T20:38:36,252 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@4f016b3e{HTTP/1.1, (http/1.1)}{0.0.0.0:33709} 2023-09-06T20:38:36,252 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @9064ms 2023-09-06T20:38:36,252 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2388): Adding backup master ZNode /1/backup-masters/jenkins-hbase3.apache.org,41705,1694032713168 2023-09-06T20:38:36,264 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-09-06T20:38:36,264 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45559-0x100ef01325b0001, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-09-06T20:38:36,266 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase3.apache.org,41705,1694032713168 2023-09-06T20:38:36,274 DEBUG [M:0;jenkins-hbase3:41705 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:41705 2023-09-06T20:38:36,286 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-09-06T20:38:36,286 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45559-0x100ef01325b0001, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-09-06T20:38:36,286 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:38:36,286 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45559-0x100ef01325b0001, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:38:36,287 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Set watcher on existing znode=/1/master 2023-09-06T20:38:36,289 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Set watcher on existing znode=/1/master 2023-09-06T20:38:36,289 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase3.apache.org,41705,1694032713168 from backup master directory 2023-09-06T20:38:36,293 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase3.apache.org,41705,1694032713168 2023-09-06T20:38:36,293 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45559-0x100ef01325b0001, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-09-06T20:38:36,293 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-09-06T20:38:36,294 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-09-06T20:38:36,294 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,41705,1694032713168 2023-09-06T20:38:36,297 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-09-06T20:38:36,299 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-09-06T20:38:36,781 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(610): Created cluster ID file at hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/hbase.id with ID: e8e6c331-0ece-42be-94f8-e93448c10844 2023-09-06T20:38:36,823 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-06T20:38:36,838 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:38:36,838 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45559-0x100ef01325b0001, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:38:36,880 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:38:36,882 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-09-06T20:38:36,901 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(270): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-09-06T20:38:36,901 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(276): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-09-06T20:38:36,903 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(285): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:283) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:310) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:38:36,913 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:38:36,914 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-09-06T20:38:36,985 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/MasterData/data/master/store 2023-09-06T20:38:37,411 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-09-06T20:38:37,412 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:38:37,412 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-09-06T20:38:37,412 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:38:37,413 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:38:37,413 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-09-06T20:38:37,413 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:38:37,413 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:38:37,413 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-09-06T20:38:37,415 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/MasterData/data/master/store/.initializing 2023-09-06T20:38:37,416 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/MasterData/WALs/jenkins-hbase3.apache.org,41705,1694032713168 2023-09-06T20:38:37,427 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-09-06T20:38:37,445 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C41705%2C1694032713168, suffix=, logDir=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/MasterData/WALs/jenkins-hbase3.apache.org,41705,1694032713168, archiveDir=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/MasterData/oldWALs, maxLogs=10 2023-09-06T20:38:37,481 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/MasterData/WALs/jenkins-hbase3.apache.org,41705,1694032713168/jenkins-hbase3.apache.org%2C41705%2C1694032713168.1694032717455, exclude list is [], retry=0 2023-09-06T20:38:37,509 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41237,DS-07aa54ab-1cb2-4831-8bde-a715b49e632a,DISK] 2023-09-06T20:38:37,517 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:346) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:430) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:424) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:482) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:477) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:489) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:38:37,586 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/MasterData/WALs/jenkins-hbase3.apache.org,41705,1694032713168/jenkins-hbase3.apache.org%2C41705%2C1694032713168.1694032717455 2023-09-06T20:38:37,586 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41237,DS-07aa54ab-1cb2-4831-8bde-a715b49e632a,DISK]] 2023-09-06T20:38:37,587 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:38:37,587 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:38:37,592 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:38:37,593 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:38:37,660 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:38:37,695 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-09-06T20:38:37,701 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:38:37,704 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:38:37,704 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:38:37,707 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-09-06T20:38:37,707 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:38:37,708 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:38:37,709 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:38:37,711 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-09-06T20:38:37,711 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:38:37,712 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:38:37,712 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:38:37,714 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-09-06T20:38:37,715 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:38:37,716 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:38:37,720 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:38:37,721 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:38:37,736 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-09-06T20:38:37,739 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-09-06T20:38:37,743 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:38:37,744 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10483568480, jitterRate=-0.02364160120487213}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-09-06T20:38:37,753 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-09-06T20:38:37,754 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-09-06T20:38:37,790 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@426dbdc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:38:37,844 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(860): No meta location available on zookeeper, skip migrating... 2023-09-06T20:38:37,865 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-09-06T20:38:37,865 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(563): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-09-06T20:38:37,868 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-09-06T20:38:37,869 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(583): Recovered RegionProcedureStore lease in 1 msec 2023-09-06T20:38:37,876 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(597): Loaded RegionProcedureStore in 6 msec 2023-09-06T20:38:37,876 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-09-06T20:38:37,913 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-09-06T20:38:37,929 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-09-06T20:38:37,931 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-09-06T20:38:37,935 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(150): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-09-06T20:38:37,936 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-09-06T20:38:37,940 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-09-06T20:38:37,943 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-09-06T20:38:37,949 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-09-06T20:38:37,951 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-09-06T20:38:37,952 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-09-06T20:38:37,953 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-09-06T20:38:37,970 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-09-06T20:38:37,972 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-09-06T20:38:37,978 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-09-06T20:38:37,978 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45559-0x100ef01325b0001, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-09-06T20:38:37,978 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45559-0x100ef01325b0001, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:38:37,978 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:38:37,979 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(804): Active/primary master=jenkins-hbase3.apache.org,41705,1694032713168, sessionid=0x100ef01325b0000, setting cluster-up flag (Was=false) 2023-09-06T20:38:37,989 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-09-06T20:38:37,996 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:38:37,996 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45559-0x100ef01325b0001, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:38:38,002 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-09-06T20:38:38,003 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,41705,1694032713168 2023-09-06T20:38:38,007 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:38:38,007 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45559-0x100ef01325b0001, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:38:38,013 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-09-06T20:38:38,014 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,41705,1694032713168 2023-09-06T20:38:38,023 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45559-0x100ef01325b0001, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:38:38,023 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:38:38,028 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-09-06T20:38:38,030 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,41705,1694032713168 2023-09-06T20:38:38,058 DEBUG [RS:0;jenkins-hbase3:45559 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:45559 2023-09-06T20:38:38,060 INFO [RS:0;jenkins-hbase3:45559 {}] regionserver.HRegionServer(997): ClusterId : e8e6c331-0ece-42be-94f8-e93448c10844 2023-09-06T20:38:38,062 INFO [RS:0;jenkins-hbase3:45559 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-09-06T20:38:38,064 DEBUG [RS:0;jenkins-hbase3:45559 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-09-06T20:38:38,071 DEBUG [RS:0;jenkins-hbase3:45559 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-09-06T20:38:38,071 DEBUG [RS:0;jenkins-hbase3:45559 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-09-06T20:38:38,074 DEBUG [RS:0;jenkins-hbase3:45559 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-09-06T20:38:38,074 DEBUG [RS:0;jenkins-hbase3:45559 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-09-06T20:38:38,077 DEBUG [RS:0;jenkins-hbase3:45559 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-09-06T20:38:38,078 DEBUG [RS:0;jenkins-hbase3:45559 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e45f88a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:38:38,081 DEBUG [RS:0;jenkins-hbase3:45559 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b79992c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-09-06T20:38:38,089 INFO [RS:0;jenkins-hbase3:45559 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-09-06T20:38:38,089 INFO [RS:0;jenkins-hbase3:45559 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-09-06T20:38:38,089 DEBUG [RS:0;jenkins-hbase3:45559 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-09-06T20:38:38,093 INFO [RS:0;jenkins-hbase3:45559 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,41705,1694032713168 with isa=jenkins-hbase3.apache.org/172.31.12.81:45559, startcode=1694032716172 2023-09-06T20:38:38,117 DEBUG [RS:0;jenkins-hbase3:45559 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-09-06T20:38:38,180 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50693, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-09-06T20:38:38,180 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-09-06T20:38:38,190 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-09-06T20:38:38,190 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41705 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3181) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:590) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-09-06T20:38:38,195 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-09-06T20:38:38,203 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,41705,1694032713168 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-09-06T20:38:38,208 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-06T20:38:38,208 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-06T20:38:38,208 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-06T20:38:38,209 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-09-06T20:38:38,209 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-09-06T20:38:38,209 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:38:38,209 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-09-06T20:38:38,209 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:38:38,212 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1694032748211 2023-09-06T20:38:38,214 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-09-06T20:38:38,219 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-09-06T20:38:38,222 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-09-06T20:38:38,223 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-09-06T20:38:38,229 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:38:38,229 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-09-06T20:38:38,230 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-09-06T20:38:38,231 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-09-06T20:38:38,231 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-09-06T20:38:38,232 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-09-06T20:38:38,232 DEBUG [RS:0;jenkins-hbase3:45559 {}] regionserver.HRegionServer(3046): Master is not running yet 2023-09-06T20:38:38,232 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-09-06T20:38:38,232 WARN [RS:0;jenkins-hbase3:45559 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 100 ms and then retrying. 2023-09-06T20:38:38,233 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-09-06T20:38:38,238 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-09-06T20:38:38,243 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-09-06T20:38:38,244 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-09-06T20:38:38,248 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-09-06T20:38:38,248 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0 2023-09-06T20:38:38,254 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7b6c9ee1 to 127.0.0.1:58035 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:38:38,270 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10081252, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:38:38,271 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-09-06T20:38:38,274 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-09-06T20:38:38,275 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-09-06T20:38:38,278 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1694032718276,5,FailOnTimeoutGroup] 2023-09-06T20:38:38,278 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1694032718278,5,FailOnTimeoutGroup] 2023-09-06T20:38:38,278 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-09-06T20:38:38,278 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1635): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-09-06T20:38:38,280 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-09-06T20:38:38,280 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-09-06T20:38:38,333 INFO [RS:0;jenkins-hbase3:45559 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,41705,1694032713168 with isa=jenkins-hbase3.apache.org/172.31.12.81:45559, startcode=1694032716172 2023-09-06T20:38:38,339 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41705 {}] master.ServerManager(403): Registering regionserver=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:38:38,350 DEBUG [RS:0;jenkins-hbase3:45559 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0 2023-09-06T20:38:38,350 DEBUG [RS:0;jenkins-hbase3:45559 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:34451 2023-09-06T20:38:38,350 DEBUG [RS:0;jenkins-hbase3:45559 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=44233 2023-09-06T20:38:38,356 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-09-06T20:38:38,357 DEBUG [RS:0;jenkins-hbase3:45559 {}] zookeeper.ZKUtil(111): regionserver:45559-0x100ef01325b0001, quorum=127.0.0.1:58035, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:38:38,357 WARN [RS:0;jenkins-hbase3:45559 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-09-06T20:38:38,357 INFO [RS:0;jenkins-hbase3:45559 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-09-06T20:38:38,357 DEBUG [RS:0;jenkins-hbase3:45559 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:38:38,360 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,45559,1694032716172] 2023-09-06T20:38:38,374 DEBUG [RS:0;jenkins-hbase3:45559 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-09-06T20:38:38,389 INFO [RS:0;jenkins-hbase3:45559 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-09-06T20:38:38,402 INFO [RS:0;jenkins-hbase3:45559 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-09-06T20:38:38,406 INFO [RS:0;jenkins-hbase3:45559 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-09-06T20:38:38,407 INFO [RS:0;jenkins-hbase3:45559 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-09-06T20:38:38,407 INFO [RS:0;jenkins-hbase3:45559 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-09-06T20:38:38,417 INFO [RS:0;jenkins-hbase3:45559 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-09-06T20:38:38,417 DEBUG [RS:0;jenkins-hbase3:45559 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:38:38,417 DEBUG [RS:0;jenkins-hbase3:45559 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:38:38,417 DEBUG [RS:0;jenkins-hbase3:45559 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:38:38,417 DEBUG [RS:0;jenkins-hbase3:45559 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:38:38,418 DEBUG [RS:0;jenkins-hbase3:45559 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:38:38,418 DEBUG [RS:0;jenkins-hbase3:45559 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-09-06T20:38:38,418 DEBUG [RS:0;jenkins-hbase3:45559 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:38:38,418 DEBUG [RS:0;jenkins-hbase3:45559 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:38:38,418 DEBUG [RS:0;jenkins-hbase3:45559 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:38:38,418 DEBUG [RS:0;jenkins-hbase3:45559 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:38:38,418 DEBUG [RS:0;jenkins-hbase3:45559 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-09-06T20:38:38,419 DEBUG [RS:0;jenkins-hbase3:45559 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-09-06T20:38:38,419 DEBUG [RS:0;jenkins-hbase3:45559 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-09-06T20:38:38,420 INFO [RS:0;jenkins-hbase3:45559 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-09-06T20:38:38,420 INFO [RS:0;jenkins-hbase3:45559 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-09-06T20:38:38,420 INFO [RS:0;jenkins-hbase3:45559 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-09-06T20:38:38,420 INFO [RS:0;jenkins-hbase3:45559 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-09-06T20:38:38,420 INFO [RS:0;jenkins-hbase3:45559 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,45559,1694032716172-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-09-06T20:38:38,437 INFO [RS:0;jenkins-hbase3:45559 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-09-06T20:38:38,440 INFO [RS:0;jenkins-hbase3:45559 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,45559,1694032716172-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-09-06T20:38:38,455 INFO [RS:0;jenkins-hbase3:45559 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,45559,1694032716172 started 2023-09-06T20:38:38,456 INFO [RS:0;jenkins-hbase3:45559 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,45559,1694032716172, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:45559, sessionid=0x100ef01325b0001 2023-09-06T20:38:38,456 DEBUG [RS:0;jenkins-hbase3:45559 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-09-06T20:38:38,456 DEBUG [RS:0;jenkins-hbase3:45559 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:38:38,456 DEBUG [RS:0;jenkins-hbase3:45559 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,45559,1694032716172' 2023-09-06T20:38:38,456 DEBUG [RS:0;jenkins-hbase3:45559 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-09-06T20:38:38,457 DEBUG [RS:0;jenkins-hbase3:45559 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-09-06T20:38:38,458 DEBUG [RS:0;jenkins-hbase3:45559 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-09-06T20:38:38,458 DEBUG [RS:0;jenkins-hbase3:45559 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-09-06T20:38:38,458 DEBUG [RS:0;jenkins-hbase3:45559 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,45559,1694032716172' 2023-09-06T20:38:38,458 DEBUG [RS:0;jenkins-hbase3:45559 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:38:38,458 DEBUG [RS:0;jenkins-hbase3:45559 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:38:38,459 INFO [RS:0;jenkins-hbase3:45559 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-09-06T20:38:38,459 DEBUG [RS:0;jenkins-hbase3:45559 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-09-06T20:38:38,459 DEBUG [RS:0;jenkins-hbase3:45559 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-09-06T20:38:38,459 DEBUG [RS:0;jenkins-hbase3:45559 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:38:38,459 DEBUG [RS:0;jenkins-hbase3:45559 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,45559,1694032716172' 2023-09-06T20:38:38,459 DEBUG [RS:0;jenkins-hbase3:45559 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-09-06T20:38:38,459 DEBUG [RS:0;jenkins-hbase3:45559 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-09-06T20:38:38,460 DEBUG [RS:0;jenkins-hbase3:45559 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-09-06T20:38:38,460 INFO [RS:0;jenkins-hbase3:45559 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-09-06T20:38:38,460 INFO [RS:0;jenkins-hbase3:45559 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-09-06T20:38:38,593 INFO [RS:0;jenkins-hbase3:45559 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-09-06T20:38:38,596 INFO [RS:0;jenkins-hbase3:45559 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C45559%2C1694032716172, suffix=, logDir=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172, archiveDir=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/oldWALs, maxLogs=32 2023-09-06T20:38:38,608 DEBUG [RS:0;jenkins-hbase3:45559 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.1694032718597, exclude list is [], retry=0 2023-09-06T20:38:38,613 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41237,DS-07aa54ab-1cb2-4831-8bde-a715b49e632a,DISK] 2023-09-06T20:38:38,617 INFO [RS:0;jenkins-hbase3:45559 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.1694032718597 2023-09-06T20:38:38,618 DEBUG [RS:0;jenkins-hbase3:45559 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41237,DS-07aa54ab-1cb2-4831-8bde-a715b49e632a,DISK]] 2023-09-06T20:38:38,668 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:38:38,671 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-09-06T20:38:38,674 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-09-06T20:38:38,674 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:38:38,675 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:38:38,675 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-09-06T20:38:38,678 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-09-06T20:38:38,678 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:38:38,679 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:38:38,679 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-09-06T20:38:38,682 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-09-06T20:38:38,682 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:38:38,683 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:38:38,684 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/hbase/meta/1588230740 2023-09-06T20:38:38,685 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/hbase/meta/1588230740 2023-09-06T20:38:38,687 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-09-06T20:38:38,689 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-09-06T20:38:38,693 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:38:38,694 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9504328320, jitterRate=-0.11484044790267944}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-09-06T20:38:38,695 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-09-06T20:38:38,696 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-09-06T20:38:38,696 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-09-06T20:38:38,696 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-09-06T20:38:38,696 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-09-06T20:38:38,696 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-09-06T20:38:38,698 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-09-06T20:38:38,698 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-09-06T20:38:38,701 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-09-06T20:38:38,701 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-09-06T20:38:38,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-09-06T20:38:38,722 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-09-06T20:38:38,726 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-09-06T20:38:38,878 DEBUG [jenkins-hbase3:41705 {}] assignment.AssignmentManager(2303): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-09-06T20:38:38,887 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:38:38,893 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,45559,1694032716172, state=OPENING 2023-09-06T20:38:38,902 DEBUG [PEWorker-4 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-09-06T20:38:38,906 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:38:38,906 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45559-0x100ef01325b0001, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:38:38,906 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-09-06T20:38:38,906 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-09-06T20:38:38,910 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,45559,1694032716172}] 2023-09-06T20:38:39,100 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(721): New admin connection to jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:38:39,103 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-09-06T20:38:39,107 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:34396, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-09-06T20:38:39,118 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-09-06T20:38:39,119 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-09-06T20:38:39,120 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-09-06T20:38:39,122 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C45559%2C1694032716172.meta, suffix=.meta, logDir=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172, archiveDir=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/oldWALs, maxLogs=32 2023-09-06T20:38:39,134 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.meta.1694032719123.meta, exclude list is [], retry=0 2023-09-06T20:38:39,138 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41237,DS-07aa54ab-1cb2-4831-8bde-a715b49e632a,DISK] 2023-09-06T20:38:39,142 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.meta.1694032719123.meta 2023-09-06T20:38:39,143 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41237,DS-07aa54ab-1cb2-4831-8bde-a715b49e632a,DISK]] 2023-09-06T20:38:39,143 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:38:39,148 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:38:39,148 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-09-06T20:38:39,166 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-09-06T20:38:39,166 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-09-06T20:38:39,171 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-09-06T20:38:39,171 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:38:39,171 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-09-06T20:38:39,172 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-09-06T20:38:39,174 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-09-06T20:38:39,176 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-09-06T20:38:39,176 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:38:39,177 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:38:39,177 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-09-06T20:38:39,178 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-09-06T20:38:39,178 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:38:39,179 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:38:39,179 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-09-06T20:38:39,180 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-09-06T20:38:39,180 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:38:39,181 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-09-06T20:38:39,182 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/hbase/meta/1588230740 2023-09-06T20:38:39,185 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/hbase/meta/1588230740 2023-09-06T20:38:39,187 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-09-06T20:38:39,189 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-09-06T20:38:39,191 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10902191520, jitterRate=0.015345707535743713}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-09-06T20:38:39,192 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-09-06T20:38:39,203 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1694032719092 2023-09-06T20:38:39,222 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:meta,,1.1588230740 2023-09-06T20:38:39,223 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-09-06T20:38:39,224 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:38:39,225 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,45559,1694032716172, state=OPEN 2023-09-06T20:38:39,229 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-09-06T20:38:39,229 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45559-0x100ef01325b0001, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-09-06T20:38:39,229 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-09-06T20:38:39,229 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-09-06T20:38:39,233 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=3, resume processing ppid=2 2023-09-06T20:38:39,234 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,45559,1694032716172 in 320 msec 2023-09-06T20:38:39,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=2, resume processing ppid=1 2023-09-06T20:38:39,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 526 msec 2023-09-06T20:38:39,384 INFO [AsyncFSWAL-0-hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/MasterData {}] wal.AbstractFSWAL(1174): Slow sync cost: 139 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41237,DS-07aa54ab-1cb2-4831-8bde-a715b49e632a,DISK]] 2023-09-06T20:38:39,390 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.2900 sec 2023-09-06T20:38:39,390 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1063): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1694032719390, completionTime=-1 2023-09-06T20:38:39,390 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(830): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-09-06T20:38:39,390 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1629): Joining cluster... 2023-09-06T20:38:39,456 DEBUG [hconnection-0x393f348c-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:38:39,459 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:34406, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:38:39,474 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1641): Number of RegionServers=1 2023-09-06T20:38:39,474 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1694032779474 2023-09-06T20:38:39,474 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1694032839474 2023-09-06T20:38:39,475 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1648): Joined the cluster in 84 msec 2023-09-06T20:38:39,497 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,41705,1694032713168-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-09-06T20:38:39,497 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,41705,1694032713168-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-09-06T20:38:39,497 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,41705,1694032713168-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-09-06T20:38:39,499 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:41705, period=300000, unit=MILLISECONDS is enabled. 2023-09-06T20:38:39,500 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-09-06T20:38:39,506 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-09-06T20:38:39,520 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-09-06T20:38:39,522 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2368): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-09-06T20:38:39,531 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-09-06T20:38:39,534 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:38:39,536 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:38:39,538 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:38:39,559 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 42b03c0f5b2d2278a9b485cc44396dc1, NAME => 'hbase:namespace,,1694032719521.42b03c0f5b2d2278a9b485cc44396dc1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0 2023-09-06T20:38:39,970 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1694032719521.42b03c0f5b2d2278a9b485cc44396dc1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:38:39,970 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing 42b03c0f5b2d2278a9b485cc44396dc1, disabling compactions & flushes 2023-09-06T20:38:39,970 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1694032719521.42b03c0f5b2d2278a9b485cc44396dc1. 2023-09-06T20:38:39,970 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1694032719521.42b03c0f5b2d2278a9b485cc44396dc1. 2023-09-06T20:38:39,971 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1694032719521.42b03c0f5b2d2278a9b485cc44396dc1. after waiting 0 ms 2023-09-06T20:38:39,971 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1694032719521.42b03c0f5b2d2278a9b485cc44396dc1. 2023-09-06T20:38:39,971 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1694032719521.42b03c0f5b2d2278a9b485cc44396dc1. 2023-09-06T20:38:39,971 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for 42b03c0f5b2d2278a9b485cc44396dc1: 2023-09-06T20:38:39,973 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:38:39,987 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1694032719521.42b03c0f5b2d2278a9b485cc44396dc1.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1694032719975"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032719975"}]},"ts":"1694032719975"} 2023-09-06T20:38:40,022 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:38:40,024 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:38:40,031 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032720024"}]},"ts":"1694032720024"} 2023-09-06T20:38:40,035 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-09-06T20:38:40,041 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=42b03c0f5b2d2278a9b485cc44396dc1, ASSIGN}] 2023-09-06T20:38:40,044 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=42b03c0f5b2d2278a9b485cc44396dc1, ASSIGN 2023-09-06T20:38:40,045 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=42b03c0f5b2d2278a9b485cc44396dc1, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45559,1694032716172; forceNewPlan=false, retain=false 2023-09-06T20:38:40,196 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=42b03c0f5b2d2278a9b485cc44396dc1, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:38:40,201 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 42b03c0f5b2d2278a9b485cc44396dc1, server=jenkins-hbase3.apache.org,45559,1694032716172}] 2023-09-06T20:38:40,361 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1694032719521.42b03c0f5b2d2278a9b485cc44396dc1. 2023-09-06T20:38:40,362 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => 42b03c0f5b2d2278a9b485cc44396dc1, NAME => 'hbase:namespace,,1694032719521.42b03c0f5b2d2278a9b485cc44396dc1.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:38:40,363 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:38:40,364 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 42b03c0f5b2d2278a9b485cc44396dc1 2023-09-06T20:38:40,364 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1694032719521.42b03c0f5b2d2278a9b485cc44396dc1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:38:40,364 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for 42b03c0f5b2d2278a9b485cc44396dc1 2023-09-06T20:38:40,364 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for 42b03c0f5b2d2278a9b485cc44396dc1 2023-09-06T20:38:40,367 INFO [StoreOpener-42b03c0f5b2d2278a9b485cc44396dc1-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 42b03c0f5b2d2278a9b485cc44396dc1 2023-09-06T20:38:40,370 INFO [StoreOpener-42b03c0f5b2d2278a9b485cc44396dc1-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42b03c0f5b2d2278a9b485cc44396dc1 columnFamilyName info 2023-09-06T20:38:40,370 DEBUG [StoreOpener-42b03c0f5b2d2278a9b485cc44396dc1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:38:40,371 INFO [StoreOpener-42b03c0f5b2d2278a9b485cc44396dc1-1 {}] regionserver.HStore(324): Store=42b03c0f5b2d2278a9b485cc44396dc1/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:38:40,373 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/hbase/namespace/42b03c0f5b2d2278a9b485cc44396dc1 2023-09-06T20:38:40,373 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/hbase/namespace/42b03c0f5b2d2278a9b485cc44396dc1 2023-09-06T20:38:40,377 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for 42b03c0f5b2d2278a9b485cc44396dc1 2023-09-06T20:38:40,382 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/hbase/namespace/42b03c0f5b2d2278a9b485cc44396dc1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:38:40,383 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened 42b03c0f5b2d2278a9b485cc44396dc1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11458707520, jitterRate=0.06717529892921448}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:38:40,384 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for 42b03c0f5b2d2278a9b485cc44396dc1: 2023-09-06T20:38:40,386 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:namespace,,1694032719521.42b03c0f5b2d2278a9b485cc44396dc1., pid=6, masterSystemTime=1694032720354 2023-09-06T20:38:40,390 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:namespace,,1694032719521.42b03c0f5b2d2278a9b485cc44396dc1. 2023-09-06T20:38:40,390 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1694032719521.42b03c0f5b2d2278a9b485cc44396dc1. 2023-09-06T20:38:40,391 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=42b03c0f5b2d2278a9b485cc44396dc1, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:38:40,398 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=6, resume processing ppid=5 2023-09-06T20:38:40,399 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 42b03c0f5b2d2278a9b485cc44396dc1, server=jenkins-hbase3.apache.org,45559,1694032716172 in 195 msec 2023-09-06T20:38:40,402 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=5, resume processing ppid=4 2023-09-06T20:38:40,402 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=42b03c0f5b2d2278a9b485cc44396dc1, ASSIGN in 358 msec 2023-09-06T20:38:40,403 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:38:40,403 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032720403"}]},"ts":"1694032720403"} 2023-09-06T20:38:40,407 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-09-06T20:38:40,409 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:38:40,412 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 887 msec 2023-09-06T20:38:40,436 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-09-06T20:38:40,437 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-09-06T20:38:40,437 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45559-0x100ef01325b0001, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:38:40,437 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-09-06T20:38:40,484 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-09-06T20:38:40,499 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:38:40,507 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 30 msec 2023-09-06T20:38:40,519 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-09-06T20:38:40,530 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:38:40,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 14 msec 2023-09-06T20:38:40,548 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-09-06T20:38:40,551 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-09-06T20:38:40,553 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1193): Master has completed initialization 4.259sec 2023-09-06T20:38:40,555 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-09-06T20:38:40,556 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-09-06T20:38:40,558 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-09-06T20:38:40,558 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-09-06T20:38:40,559 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-09-06T20:38:40,560 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,41705,1694032713168-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-09-06T20:38:40,561 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,41705,1694032713168-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-09-06T20:38:40,574 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1296): Balancer post startup initialization complete, took 0 seconds 2023-09-06T20:38:40,575 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-09-06T20:38:40,589 DEBUG [Listener at localhost/38777 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x03ca2a7b to 127.0.0.1:58035 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:38:40,594 DEBUG [Listener at localhost/38777 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@442db8ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:38:40,597 DEBUG [Listener at localhost/38777 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-09-06T20:38:40,597 DEBUG [Listener at localhost/38777 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-09-06T20:38:40,611 DEBUG [hconnection-0xd3d4503-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:38:40,622 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:56958, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:38:40,632 INFO [Listener at localhost/38777 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,41705,1694032713168 2023-09-06T20:38:40,633 INFO [Listener at localhost/38777 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-09-06T20:38:40,633 INFO [Listener at localhost/38777 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/test.cache.data in system properties and HBase conf 2023-09-06T20:38:40,633 INFO [Listener at localhost/38777 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/hadoop.tmp.dir in system properties and HBase conf 2023-09-06T20:38:40,633 INFO [Listener at localhost/38777 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/hadoop.log.dir in system properties and HBase conf 2023-09-06T20:38:40,633 INFO [Listener at localhost/38777 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/mapreduce.cluster.local.dir in system properties and HBase conf 2023-09-06T20:38:40,633 INFO [Listener at localhost/38777 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-09-06T20:38:40,633 INFO [Listener at localhost/38777 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-09-06T20:38:40,633 INFO [Listener at localhost/38777 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-09-06T20:38:40,633 INFO [Listener at localhost/38777 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-09-06T20:38:40,633 INFO [Listener at localhost/38777 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-09-06T20:38:40,633 INFO [Listener at localhost/38777 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-06T20:38:40,633 INFO [Listener at localhost/38777 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-09-06T20:38:40,634 INFO [Listener at localhost/38777 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-09-06T20:38:40,634 INFO [Listener at localhost/38777 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-09-06T20:38:40,634 INFO [Listener at localhost/38777 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-06T20:38:40,634 INFO [Listener at localhost/38777 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-09-06T20:38:40,634 INFO [Listener at localhost/38777 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/nfs.dump.dir in system properties and HBase conf 2023-09-06T20:38:40,634 INFO [Listener at localhost/38777 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/java.io.tmpdir in system properties and HBase conf 2023-09-06T20:38:40,634 INFO [Listener at localhost/38777 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/dfs.journalnode.edits.dir in system properties and HBase conf 2023-09-06T20:38:40,634 INFO [Listener at localhost/38777 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-09-06T20:38:40,634 INFO [Listener at localhost/38777 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-09-06T20:38:42,086 WARN [Thread-249 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:38:42,448 INFO [Thread-249 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:38:42,454 INFO [Thread-249 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_43897_jobhistory____un8vwh/webapp 2023-09-06T20:38:42,626 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-09-06T20:38:42,626 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-09-06T20:38:42,626 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-09-06T20:38:42,630 INFO [Thread-249 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-09-06T20:38:42,723 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:38:42,849 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:38:43,140 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-09-06T20:38:43,169 INFO [Thread-249 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:43897 2023-09-06T20:38:43,581 WARN [Listener at jenkins-hbase3.apache.org/41055 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:38:43,662 WARN [Listener at jenkins-hbase3.apache.org/41055 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-09-06T20:38:43,663 INFO [Listener at jenkins-hbase3.apache.org/41055 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:38:43,670 INFO [Listener at jenkins-hbase3.apache.org/41055 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_42093_cluster____.w0s19l/webapp 2023-09-06T20:38:43,759 INFO [Listener at jenkins-hbase3.apache.org/41055 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-09-06T20:38:43,759 INFO [Listener at jenkins-hbase3.apache.org/41055 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-09-06T20:38:43,759 INFO [Listener at jenkins-hbase3.apache.org/41055 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-09-06T20:38:43,760 INFO [Listener at jenkins-hbase3.apache.org/41055 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-09-06T20:38:43,772 INFO [Listener at jenkins-hbase3.apache.org/41055 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:38:43,988 INFO [Listener at jenkins-hbase3.apache.org/41055 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:38:44,089 INFO [Listener at jenkins-hbase3.apache.org/41055 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:38:44,097 INFO [Listener at jenkins-hbase3.apache.org/41055 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:42093 2023-09-06T20:38:44,490 WARN [Listener at jenkins-hbase3.apache.org/33871 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:38:44,514 WARN [Listener at jenkins-hbase3.apache.org/33871 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-09-06T20:38:44,514 INFO [Listener at jenkins-hbase3.apache.org/33871 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:38:44,521 INFO [Listener at jenkins-hbase3.apache.org/33871 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_45093_node____941x6w/webapp 2023-09-06T20:38:44,575 INFO [Listener at jenkins-hbase3.apache.org/33871 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-09-06T20:38:44,575 INFO [Listener at jenkins-hbase3.apache.org/33871 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-09-06T20:38:44,575 INFO [Listener at jenkins-hbase3.apache.org/33871 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-09-06T20:38:44,575 INFO [Listener at jenkins-hbase3.apache.org/33871 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-09-06T20:38:44,584 INFO [Listener at jenkins-hbase3.apache.org/33871 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:38:44,608 INFO [Listener at jenkins-hbase3.apache.org/33871 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:38:44,683 INFO [Listener at jenkins-hbase3.apache.org/33871 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:38:44,693 INFO [Listener at jenkins-hbase3.apache.org/33871 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:45093 2023-09-06T20:38:44,710 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:38:44,823 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-09-06T20:38:44,824 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-09-06T20:38:44,833 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-09-06T20:38:44,833 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-09-06T20:38:44,842 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:38:44,842 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-09-06T20:38:44,843 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-09-06T20:38:44,843 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-09-06T20:38:44,843 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-09-06T20:38:44,843 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-09-06T20:38:44,848 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-09-06T20:38:44,852 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-09-06T20:38:44,941 WARN [Listener at jenkins-hbase3.apache.org/41977 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-09-06T20:38:44,948 WARN [Listener at jenkins-hbase3.apache.org/41977 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-09-06T20:38:44,949 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] log.Slf4jLog(67): jetty-6.1.26 2023-09-06T20:38:44,956 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_45749_node____.6v2fie/webapp 2023-09-06T20:38:45,006 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-09-06T20:38:45,006 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-09-06T20:38:45,006 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-09-06T20:38:45,006 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-09-06T20:38:45,017 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:38:45,043 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:38:45,115 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-09-06T20:38:45,122 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:45749 2023-09-06T20:38:45,152 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-09-06T20:38:45,153 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:34451/backupUT 2023-09-06T20:38:45,165 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:38:45,169 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49130, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:38:45,180 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns1'} 2023-09-06T20:38:45,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-09-06T20:38:45,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-09-06T20:38:45,230 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:38:45,235 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 53 msec 2023-09-06T20:38:45,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-09-06T20:38:45,326 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns2'} 2023-09-06T20:38:45,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-09-06T20:38:45,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=10 2023-09-06T20:38:45,341 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:38:45,347 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 20 msec 2023-09-06T20:38:45,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=10 2023-09-06T20:38:45,436 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns3'} 2023-09-06T20:38:45,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-09-06T20:38:45,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=11 2023-09-06T20:38:45,450 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:38:45,455 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 17 msec 2023-09-06T20:38:45,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=11 2023-09-06T20:38:45,548 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns4'} 2023-09-06T20:38:45,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-09-06T20:38:45,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-09-06T20:38:45,562 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:38:45,567 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 17 msec 2023-09-06T20:38:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-09-06T20:38:45,670 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'test-1694032725153', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:38:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1694032725153 2023-09-06T20:38:45,674 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1694032725153 execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:38:45,675 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:38:45,676 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1694032725153 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:38:45,680 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "test-1694032725153" procId is: 13 2023-09-06T20:38:45,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-09-06T20:38:45,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-09-06T20:38:45,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-09-06T20:38:46,108 INFO [RegionOpenAndInit-test-1694032725153-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 7499612530301bea74e2d6f06f601777, NAME => 'test-1694032725153,,1694032725664.7499612530301bea74e2d6f06f601777.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1694032725153', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0 2023-09-06T20:38:46,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-09-06T20:38:46,532 DEBUG [RegionOpenAndInit-test-1694032725153-pool-0 {}] regionserver.HRegion(888): Instantiated test-1694032725153,,1694032725664.7499612530301bea74e2d6f06f601777.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:38:46,532 DEBUG [RegionOpenAndInit-test-1694032725153-pool-0 {}] regionserver.HRegion(1668): Closing 7499612530301bea74e2d6f06f601777, disabling compactions & flushes 2023-09-06T20:38:46,532 INFO [RegionOpenAndInit-test-1694032725153-pool-0 {}] regionserver.HRegion(1690): Closing region test-1694032725153,,1694032725664.7499612530301bea74e2d6f06f601777. 2023-09-06T20:38:46,532 DEBUG [RegionOpenAndInit-test-1694032725153-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1694032725153,,1694032725664.7499612530301bea74e2d6f06f601777. 2023-09-06T20:38:46,532 DEBUG [RegionOpenAndInit-test-1694032725153-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1694032725153,,1694032725664.7499612530301bea74e2d6f06f601777. after waiting 0 ms 2023-09-06T20:38:46,532 DEBUG [RegionOpenAndInit-test-1694032725153-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1694032725153,,1694032725664.7499612530301bea74e2d6f06f601777. 2023-09-06T20:38:46,532 INFO [RegionOpenAndInit-test-1694032725153-pool-0 {}] regionserver.HRegion(1909): Closed test-1694032725153,,1694032725664.7499612530301bea74e2d6f06f601777. 2023-09-06T20:38:46,532 DEBUG [RegionOpenAndInit-test-1694032725153-pool-0 {}] regionserver.HRegion(1622): Region close journal for 7499612530301bea74e2d6f06f601777: 2023-09-06T20:38:46,534 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1694032725153 execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:38:46,534 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1694032725153,,1694032725664.7499612530301bea74e2d6f06f601777.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1694032726534"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032726534"}]},"ts":"1694032726534"} 2023-09-06T20:38:46,537 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:38:46,538 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1694032725153 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:38:46,539 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1694032725153","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032726538"}]},"ts":"1694032726538"} 2023-09-06T20:38:46,541 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1694032725153, state=ENABLING in hbase:meta 2023-09-06T20:38:46,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1694032725153, region=7499612530301bea74e2d6f06f601777, ASSIGN}] 2023-09-06T20:38:46,548 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1694032725153, region=7499612530301bea74e2d6f06f601777, ASSIGN 2023-09-06T20:38:46,549 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1694032725153, region=7499612530301bea74e2d6f06f601777, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45559,1694032716172; forceNewPlan=false, retain=false 2023-09-06T20:38:46,701 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=7499612530301bea74e2d6f06f601777, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:38:46,704 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure 7499612530301bea74e2d6f06f601777, server=jenkins-hbase3.apache.org,45559,1694032716172}] 2023-09-06T20:38:46,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-09-06T20:38:46,865 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1694032725153,,1694032725664.7499612530301bea74e2d6f06f601777. 2023-09-06T20:38:46,865 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => 7499612530301bea74e2d6f06f601777, NAME => 'test-1694032725153,,1694032725664.7499612530301bea74e2d6f06f601777.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:38:46,866 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:38:46,866 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1694032725153 7499612530301bea74e2d6f06f601777 2023-09-06T20:38:46,866 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1694032725153,,1694032725664.7499612530301bea74e2d6f06f601777.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:38:46,866 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for 7499612530301bea74e2d6f06f601777 2023-09-06T20:38:46,866 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for 7499612530301bea74e2d6f06f601777 2023-09-06T20:38:46,869 INFO [StoreOpener-7499612530301bea74e2d6f06f601777-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 7499612530301bea74e2d6f06f601777 2023-09-06T20:38:46,871 INFO [StoreOpener-7499612530301bea74e2d6f06f601777-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7499612530301bea74e2d6f06f601777 columnFamilyName f 2023-09-06T20:38:46,871 DEBUG [StoreOpener-7499612530301bea74e2d6f06f601777-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:38:46,872 INFO [StoreOpener-7499612530301bea74e2d6f06f601777-1 {}] regionserver.HStore(324): Store=7499612530301bea74e2d6f06f601777/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:38:46,873 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/default/test-1694032725153/7499612530301bea74e2d6f06f601777 2023-09-06T20:38:46,874 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/default/test-1694032725153/7499612530301bea74e2d6f06f601777 2023-09-06T20:38:46,878 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for 7499612530301bea74e2d6f06f601777 2023-09-06T20:38:46,886 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/default/test-1694032725153/7499612530301bea74e2d6f06f601777/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:38:46,887 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened 7499612530301bea74e2d6f06f601777; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9431334560, jitterRate=-0.12163852155208588}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:38:46,888 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for 7499612530301bea74e2d6f06f601777: 2023-09-06T20:38:46,890 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1694032725153,,1694032725664.7499612530301bea74e2d6f06f601777., pid=15, masterSystemTime=1694032726857 2023-09-06T20:38:46,893 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1694032725153,,1694032725664.7499612530301bea74e2d6f06f601777. 2023-09-06T20:38:46,893 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1694032725153,,1694032725664.7499612530301bea74e2d6f06f601777. 2023-09-06T20:38:46,894 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=7499612530301bea74e2d6f06f601777, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:38:46,900 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=15, resume processing ppid=14 2023-09-06T20:38:46,900 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure 7499612530301bea74e2d6f06f601777, server=jenkins-hbase3.apache.org,45559,1694032716172 in 193 msec 2023-09-06T20:38:46,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=14, resume processing ppid=13 2023-09-06T20:38:46,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1694032725153, region=7499612530301bea74e2d6f06f601777, ASSIGN in 354 msec 2023-09-06T20:38:46,904 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1694032725153 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:38:46,904 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1694032725153","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032726904"}]},"ts":"1694032726904"} 2023-09-06T20:38:46,907 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1694032725153, state=ENABLED in hbase:meta 2023-09-06T20:38:46,910 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1694032725153 execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:38:46,913 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1694032725153 in 1.2400 sec 2023-09-06T20:38:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-09-06T20:38:47,799 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:test-1694032725153, procId: 13 completed 2023-09-06T20:38:47,801 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x38df351f to 127.0.0.1:58035 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-09-06T20:38:47,806 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57a0a272, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:38:47,809 DEBUG [hconnection-0x4f398338-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:38:47,812 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:56964, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:38:47,817 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] regionserver.HRegion(8210): writing data to region test-1694032725153,,1694032725664.7499612530301bea74e2d6f06f601777. with WAL disabled. Data may be lost in the event of a crash. 2023-09-06T20:38:47,924 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns2:test-16940327251531', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:38:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-16940327251531 2023-09-06T20:38:47,928 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16940327251531 execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:38:47,928 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:38:47,928 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "test-16940327251531" procId is: 16 2023-09-06T20:38:47,929 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-16940327251531 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:38:47,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:38:47,946 INFO [RegionOpenAndInit-ns2:test-16940327251531-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 87e84cabad1c9957eead562b206d922d, NAME => 'ns2:test-16940327251531,,1694032727923.87e84cabad1c9957eead562b206d922d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16940327251531', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0 2023-09-06T20:38:48,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:38:48,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:38:48,359 DEBUG [RegionOpenAndInit-ns2:test-16940327251531-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16940327251531,,1694032727923.87e84cabad1c9957eead562b206d922d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:38:48,359 DEBUG [RegionOpenAndInit-ns2:test-16940327251531-pool-0 {}] regionserver.HRegion(1668): Closing 87e84cabad1c9957eead562b206d922d, disabling compactions & flushes 2023-09-06T20:38:48,360 INFO [RegionOpenAndInit-ns2:test-16940327251531-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16940327251531,,1694032727923.87e84cabad1c9957eead562b206d922d. 2023-09-06T20:38:48,360 DEBUG [RegionOpenAndInit-ns2:test-16940327251531-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16940327251531,,1694032727923.87e84cabad1c9957eead562b206d922d. 2023-09-06T20:38:48,360 DEBUG [RegionOpenAndInit-ns2:test-16940327251531-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16940327251531,,1694032727923.87e84cabad1c9957eead562b206d922d. after waiting 0 ms 2023-09-06T20:38:48,360 DEBUG [RegionOpenAndInit-ns2:test-16940327251531-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16940327251531,,1694032727923.87e84cabad1c9957eead562b206d922d. 2023-09-06T20:38:48,360 INFO [RegionOpenAndInit-ns2:test-16940327251531-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16940327251531,,1694032727923.87e84cabad1c9957eead562b206d922d. 2023-09-06T20:38:48,360 DEBUG [RegionOpenAndInit-ns2:test-16940327251531-pool-0 {}] regionserver.HRegion(1622): Region close journal for 87e84cabad1c9957eead562b206d922d: 2023-09-06T20:38:48,361 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-16940327251531 execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:38:48,362 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16940327251531,,1694032727923.87e84cabad1c9957eead562b206d922d.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1694032728361"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032728361"}]},"ts":"1694032728361"} 2023-09-06T20:38:48,364 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:38:48,365 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-16940327251531 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:38:48,365 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16940327251531","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032728365"}]},"ts":"1694032728365"} 2023-09-06T20:38:48,367 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16940327251531, state=ENABLING in hbase:meta 2023-09-06T20:38:48,372 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16940327251531, region=87e84cabad1c9957eead562b206d922d, ASSIGN}] 2023-09-06T20:38:48,374 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16940327251531, region=87e84cabad1c9957eead562b206d922d, ASSIGN 2023-09-06T20:38:48,375 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16940327251531, region=87e84cabad1c9957eead562b206d922d, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45559,1694032716172; forceNewPlan=false, retain=false 2023-09-06T20:38:48,527 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=87e84cabad1c9957eead562b206d922d, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:38:48,530 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure 87e84cabad1c9957eead562b206d922d, server=jenkins-hbase3.apache.org,45559,1694032716172}] 2023-09-06T20:38:48,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:38:48,687 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-16940327251531,,1694032727923.87e84cabad1c9957eead562b206d922d. 2023-09-06T20:38:48,687 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => 87e84cabad1c9957eead562b206d922d, NAME => 'ns2:test-16940327251531,,1694032727923.87e84cabad1c9957eead562b206d922d.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:38:48,687 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:38:48,687 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16940327251531 87e84cabad1c9957eead562b206d922d 2023-09-06T20:38:48,687 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-16940327251531,,1694032727923.87e84cabad1c9957eead562b206d922d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:38:48,687 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for 87e84cabad1c9957eead562b206d922d 2023-09-06T20:38:48,687 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for 87e84cabad1c9957eead562b206d922d 2023-09-06T20:38:48,689 INFO [StoreOpener-87e84cabad1c9957eead562b206d922d-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 87e84cabad1c9957eead562b206d922d 2023-09-06T20:38:48,691 INFO [StoreOpener-87e84cabad1c9957eead562b206d922d-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87e84cabad1c9957eead562b206d922d columnFamilyName f 2023-09-06T20:38:48,692 DEBUG [StoreOpener-87e84cabad1c9957eead562b206d922d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:38:48,692 INFO [StoreOpener-87e84cabad1c9957eead562b206d922d-1 {}] regionserver.HStore(324): Store=87e84cabad1c9957eead562b206d922d/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:38:48,694 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/ns2/test-16940327251531/87e84cabad1c9957eead562b206d922d 2023-09-06T20:38:48,694 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/ns2/test-16940327251531/87e84cabad1c9957eead562b206d922d 2023-09-06T20:38:48,697 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for 87e84cabad1c9957eead562b206d922d 2023-09-06T20:38:48,704 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/ns2/test-16940327251531/87e84cabad1c9957eead562b206d922d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:38:48,705 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened 87e84cabad1c9957eead562b206d922d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11145738880, jitterRate=0.038027822971343994}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:38:48,705 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for 87e84cabad1c9957eead562b206d922d: 2023-09-06T20:38:48,706 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:test-16940327251531,,1694032727923.87e84cabad1c9957eead562b206d922d., pid=18, masterSystemTime=1694032728682 2023-09-06T20:38:48,709 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:test-16940327251531,,1694032727923.87e84cabad1c9957eead562b206d922d. 2023-09-06T20:38:48,709 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-16940327251531,,1694032727923.87e84cabad1c9957eead562b206d922d. 2023-09-06T20:38:48,710 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=87e84cabad1c9957eead562b206d922d, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:38:48,715 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=18, resume processing ppid=17 2023-09-06T20:38:48,715 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure 87e84cabad1c9957eead562b206d922d, server=jenkins-hbase3.apache.org,45559,1694032716172 in 183 msec 2023-09-06T20:38:48,717 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=17, resume processing ppid=16 2023-09-06T20:38:48,717 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16940327251531, region=87e84cabad1c9957eead562b206d922d, ASSIGN in 343 msec 2023-09-06T20:38:48,718 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-16940327251531 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:38:48,719 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16940327251531","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032728718"}]},"ts":"1694032728718"} 2023-09-06T20:38:48,720 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16940327251531, state=ENABLED in hbase:meta 2023-09-06T20:38:48,723 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16940327251531 execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:38:48,725 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-16940327251531 in 799 msec 2023-09-06T20:38:49,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-09-06T20:38:49,037 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns2:test-16940327251531, procId: 16 completed 2023-09-06T20:38:49,041 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] regionserver.HRegion(8210): writing data to region ns2:test-16940327251531,,1694032727923.87e84cabad1c9957eead562b206d922d. with WAL disabled. Data may be lost in the event of a crash. 2023-09-06T20:38:49,129 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns3:test-16940327251532', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:38:49,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-16940327251532 2023-09-06T20:38:49,133 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16940327251532 execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:38:49,133 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:38:49,133 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "test-16940327251532" procId is: 19 2023-09-06T20:38:49,134 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-16940327251532 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:38:49,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-09-06T20:38:49,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-09-06T20:38:49,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-09-06T20:38:49,551 INFO [RegionOpenAndInit-ns3:test-16940327251532-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => b41ddfc19cb7ed42d2519d4bd835b296, NAME => 'ns3:test-16940327251532,,1694032729129.b41ddfc19cb7ed42d2519d4bd835b296.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16940327251532', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0 2023-09-06T20:38:49,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-09-06T20:38:49,969 DEBUG [RegionOpenAndInit-ns3:test-16940327251532-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16940327251532,,1694032729129.b41ddfc19cb7ed42d2519d4bd835b296.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:38:49,969 DEBUG [RegionOpenAndInit-ns3:test-16940327251532-pool-0 {}] regionserver.HRegion(1668): Closing b41ddfc19cb7ed42d2519d4bd835b296, disabling compactions & flushes 2023-09-06T20:38:49,969 INFO [RegionOpenAndInit-ns3:test-16940327251532-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16940327251532,,1694032729129.b41ddfc19cb7ed42d2519d4bd835b296. 2023-09-06T20:38:49,969 DEBUG [RegionOpenAndInit-ns3:test-16940327251532-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16940327251532,,1694032729129.b41ddfc19cb7ed42d2519d4bd835b296. 2023-09-06T20:38:49,969 DEBUG [RegionOpenAndInit-ns3:test-16940327251532-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16940327251532,,1694032729129.b41ddfc19cb7ed42d2519d4bd835b296. after waiting 0 ms 2023-09-06T20:38:49,969 DEBUG [RegionOpenAndInit-ns3:test-16940327251532-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16940327251532,,1694032729129.b41ddfc19cb7ed42d2519d4bd835b296. 2023-09-06T20:38:49,969 INFO [RegionOpenAndInit-ns3:test-16940327251532-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16940327251532,,1694032729129.b41ddfc19cb7ed42d2519d4bd835b296. 2023-09-06T20:38:49,969 DEBUG [RegionOpenAndInit-ns3:test-16940327251532-pool-0 {}] regionserver.HRegion(1622): Region close journal for b41ddfc19cb7ed42d2519d4bd835b296: 2023-09-06T20:38:49,971 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-16940327251532 execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:38:49,971 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16940327251532,,1694032729129.b41ddfc19cb7ed42d2519d4bd835b296.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1694032729971"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032729971"}]},"ts":"1694032729971"} 2023-09-06T20:38:49,974 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:38:49,975 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-16940327251532 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:38:49,975 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16940327251532","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032729975"}]},"ts":"1694032729975"} 2023-09-06T20:38:49,977 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16940327251532, state=ENABLING in hbase:meta 2023-09-06T20:38:49,982 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16940327251532, region=b41ddfc19cb7ed42d2519d4bd835b296, ASSIGN}] 2023-09-06T20:38:49,984 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16940327251532, region=b41ddfc19cb7ed42d2519d4bd835b296, ASSIGN 2023-09-06T20:38:49,985 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16940327251532, region=b41ddfc19cb7ed42d2519d4bd835b296, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45559,1694032716172; forceNewPlan=false, retain=false 2023-09-06T20:38:50,137 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=b41ddfc19cb7ed42d2519d4bd835b296, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:38:50,140 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure b41ddfc19cb7ed42d2519d4bd835b296, server=jenkins-hbase3.apache.org,45559,1694032716172}] 2023-09-06T20:38:50,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-09-06T20:38:50,298 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-16940327251532,,1694032729129.b41ddfc19cb7ed42d2519d4bd835b296. 2023-09-06T20:38:50,299 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => b41ddfc19cb7ed42d2519d4bd835b296, NAME => 'ns3:test-16940327251532,,1694032729129.b41ddfc19cb7ed42d2519d4bd835b296.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:38:50,299 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:38:50,299 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16940327251532 b41ddfc19cb7ed42d2519d4bd835b296 2023-09-06T20:38:50,299 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-16940327251532,,1694032729129.b41ddfc19cb7ed42d2519d4bd835b296.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:38:50,299 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for b41ddfc19cb7ed42d2519d4bd835b296 2023-09-06T20:38:50,299 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for b41ddfc19cb7ed42d2519d4bd835b296 2023-09-06T20:38:50,302 INFO [StoreOpener-b41ddfc19cb7ed42d2519d4bd835b296-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region b41ddfc19cb7ed42d2519d4bd835b296 2023-09-06T20:38:50,304 INFO [StoreOpener-b41ddfc19cb7ed42d2519d4bd835b296-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b41ddfc19cb7ed42d2519d4bd835b296 columnFamilyName f 2023-09-06T20:38:50,304 DEBUG [StoreOpener-b41ddfc19cb7ed42d2519d4bd835b296-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:38:50,305 INFO [StoreOpener-b41ddfc19cb7ed42d2519d4bd835b296-1 {}] regionserver.HStore(324): Store=b41ddfc19cb7ed42d2519d4bd835b296/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:38:50,306 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/ns3/test-16940327251532/b41ddfc19cb7ed42d2519d4bd835b296 2023-09-06T20:38:50,307 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/ns3/test-16940327251532/b41ddfc19cb7ed42d2519d4bd835b296 2023-09-06T20:38:50,314 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for b41ddfc19cb7ed42d2519d4bd835b296 2023-09-06T20:38:50,324 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/ns3/test-16940327251532/b41ddfc19cb7ed42d2519d4bd835b296/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:38:50,325 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened b41ddfc19cb7ed42d2519d4bd835b296; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9758192960, jitterRate=-0.09119746088981628}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:38:50,326 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for b41ddfc19cb7ed42d2519d4bd835b296: 2023-09-06T20:38:50,328 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:test-16940327251532,,1694032729129.b41ddfc19cb7ed42d2519d4bd835b296., pid=21, masterSystemTime=1694032730293 2023-09-06T20:38:50,330 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:test-16940327251532,,1694032729129.b41ddfc19cb7ed42d2519d4bd835b296. 2023-09-06T20:38:50,331 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-16940327251532,,1694032729129.b41ddfc19cb7ed42d2519d4bd835b296. 2023-09-06T20:38:50,331 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=b41ddfc19cb7ed42d2519d4bd835b296, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:38:50,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=21, resume processing ppid=20 2023-09-06T20:38:50,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure b41ddfc19cb7ed42d2519d4bd835b296, server=jenkins-hbase3.apache.org,45559,1694032716172 in 194 msec 2023-09-06T20:38:50,340 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=20, resume processing ppid=19 2023-09-06T20:38:50,340 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16940327251532, region=b41ddfc19cb7ed42d2519d4bd835b296, ASSIGN in 355 msec 2023-09-06T20:38:50,341 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-16940327251532 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:38:50,341 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16940327251532","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032730341"}]},"ts":"1694032730341"} 2023-09-06T20:38:50,343 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16940327251532, state=ENABLED in hbase:meta 2023-09-06T20:38:50,347 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16940327251532 execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:38:50,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-16940327251532 in 1.2180 sec 2023-09-06T20:38:50,854 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-16940327251531' 2023-09-06T20:38:50,854 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1694032725153' 2023-09-06T20:38:50,855 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16940327251532' 2023-09-06T20:38:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-09-06T20:38:51,245 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns3:test-16940327251532, procId: 19 completed 2023-09-06T20:38:51,245 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-16940327251532 get assigned. Timeout = 60000ms 2023-09-06T20:38:51,247 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-06T20:38:51,252 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-16940327251532 assigned to meta. Checking AM states. 2023-09-06T20:38:51,253 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-06T20:38:51,253 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-16940327251532 assigned. 2023-09-06T20:38:51,255 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns4:test-16940327251533', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:38:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-16940327251533 2023-09-06T20:38:51,259 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16940327251533 execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:38:51,259 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:38:51,259 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns4" qualifier: "test-16940327251533" procId is: 22 2023-09-06T20:38:51,260 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-16940327251533 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:38:51,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:38:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:38:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:38:51,675 INFO [RegionOpenAndInit-ns4:test-16940327251533-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 17f2e8bf9b7a35cfadd958888b7227bd, NAME => 'ns4:test-16940327251533,,1694032731255.17f2e8bf9b7a35cfadd958888b7227bd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-16940327251533', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0 2023-09-06T20:38:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:38:52,090 DEBUG [RegionOpenAndInit-ns4:test-16940327251533-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-16940327251533,,1694032731255.17f2e8bf9b7a35cfadd958888b7227bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:38:52,090 DEBUG [RegionOpenAndInit-ns4:test-16940327251533-pool-0 {}] regionserver.HRegion(1668): Closing 17f2e8bf9b7a35cfadd958888b7227bd, disabling compactions & flushes 2023-09-06T20:38:52,090 INFO [RegionOpenAndInit-ns4:test-16940327251533-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-16940327251533,,1694032731255.17f2e8bf9b7a35cfadd958888b7227bd. 2023-09-06T20:38:52,090 DEBUG [RegionOpenAndInit-ns4:test-16940327251533-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16940327251533,,1694032731255.17f2e8bf9b7a35cfadd958888b7227bd. 2023-09-06T20:38:52,090 DEBUG [RegionOpenAndInit-ns4:test-16940327251533-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16940327251533,,1694032731255.17f2e8bf9b7a35cfadd958888b7227bd. after waiting 0 ms 2023-09-06T20:38:52,090 DEBUG [RegionOpenAndInit-ns4:test-16940327251533-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16940327251533,,1694032731255.17f2e8bf9b7a35cfadd958888b7227bd. 2023-09-06T20:38:52,090 INFO [RegionOpenAndInit-ns4:test-16940327251533-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-16940327251533,,1694032731255.17f2e8bf9b7a35cfadd958888b7227bd. 2023-09-06T20:38:52,090 DEBUG [RegionOpenAndInit-ns4:test-16940327251533-pool-0 {}] regionserver.HRegion(1622): Region close journal for 17f2e8bf9b7a35cfadd958888b7227bd: 2023-09-06T20:38:52,092 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-16940327251533 execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:38:52,092 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-16940327251533,,1694032731255.17f2e8bf9b7a35cfadd958888b7227bd.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1694032732092"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032732092"}]},"ts":"1694032732092"} 2023-09-06T20:38:52,094 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:38:52,095 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-16940327251533 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:38:52,095 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16940327251533","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032732095"}]},"ts":"1694032732095"} 2023-09-06T20:38:52,097 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16940327251533, state=ENABLING in hbase:meta 2023-09-06T20:38:52,102 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16940327251533, region=17f2e8bf9b7a35cfadd958888b7227bd, ASSIGN}] 2023-09-06T20:38:52,104 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16940327251533, region=17f2e8bf9b7a35cfadd958888b7227bd, ASSIGN 2023-09-06T20:38:52,105 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-16940327251533, region=17f2e8bf9b7a35cfadd958888b7227bd, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45559,1694032716172; forceNewPlan=false, retain=false 2023-09-06T20:38:52,256 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=17f2e8bf9b7a35cfadd958888b7227bd, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:38:52,259 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure 17f2e8bf9b7a35cfadd958888b7227bd, server=jenkins-hbase3.apache.org,45559,1694032716172}] 2023-09-06T20:38:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:38:52,375 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:38:52,417 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-16940327251533,,1694032731255.17f2e8bf9b7a35cfadd958888b7227bd. 2023-09-06T20:38:52,417 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => 17f2e8bf9b7a35cfadd958888b7227bd, NAME => 'ns4:test-16940327251533,,1694032731255.17f2e8bf9b7a35cfadd958888b7227bd.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:38:52,417 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:38:52,418 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16940327251533 17f2e8bf9b7a35cfadd958888b7227bd 2023-09-06T20:38:52,418 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-16940327251533,,1694032731255.17f2e8bf9b7a35cfadd958888b7227bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:38:52,418 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for 17f2e8bf9b7a35cfadd958888b7227bd 2023-09-06T20:38:52,418 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for 17f2e8bf9b7a35cfadd958888b7227bd 2023-09-06T20:38:52,422 INFO [StoreOpener-17f2e8bf9b7a35cfadd958888b7227bd-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 17f2e8bf9b7a35cfadd958888b7227bd 2023-09-06T20:38:52,429 INFO [StoreOpener-17f2e8bf9b7a35cfadd958888b7227bd-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17f2e8bf9b7a35cfadd958888b7227bd columnFamilyName f 2023-09-06T20:38:52,429 DEBUG [StoreOpener-17f2e8bf9b7a35cfadd958888b7227bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:38:52,432 INFO [StoreOpener-17f2e8bf9b7a35cfadd958888b7227bd-1 {}] regionserver.HStore(324): Store=17f2e8bf9b7a35cfadd958888b7227bd/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:38:52,434 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/ns4/test-16940327251533/17f2e8bf9b7a35cfadd958888b7227bd 2023-09-06T20:38:52,434 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/ns4/test-16940327251533/17f2e8bf9b7a35cfadd958888b7227bd 2023-09-06T20:38:52,440 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for 17f2e8bf9b7a35cfadd958888b7227bd 2023-09-06T20:38:52,447 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/ns4/test-16940327251533/17f2e8bf9b7a35cfadd958888b7227bd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:38:52,448 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened 17f2e8bf9b7a35cfadd958888b7227bd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11514534880, jitterRate=0.072374626994133}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-06T20:38:52,448 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for 17f2e8bf9b7a35cfadd958888b7227bd: 2023-09-06T20:38:52,450 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2556): Post open deploy tasks for ns4:test-16940327251533,,1694032731255.17f2e8bf9b7a35cfadd958888b7227bd., pid=24, masterSystemTime=1694032732411 2023-09-06T20:38:52,455 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2583): Finished post open deploy task for ns4:test-16940327251533,,1694032731255.17f2e8bf9b7a35cfadd958888b7227bd. 2023-09-06T20:38:52,455 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-16940327251533,,1694032731255.17f2e8bf9b7a35cfadd958888b7227bd. 2023-09-06T20:38:52,456 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=17f2e8bf9b7a35cfadd958888b7227bd, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:38:52,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=24, resume processing ppid=23 2023-09-06T20:38:52,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure 17f2e8bf9b7a35cfadd958888b7227bd, server=jenkins-hbase3.apache.org,45559,1694032716172 in 203 msec 2023-09-06T20:38:52,469 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=23, resume processing ppid=22 2023-09-06T20:38:52,469 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-16940327251533, region=17f2e8bf9b7a35cfadd958888b7227bd, ASSIGN in 364 msec 2023-09-06T20:38:52,470 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-16940327251533 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:38:52,470 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16940327251533","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032732470"}]},"ts":"1694032732470"} 2023-09-06T20:38:52,472 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16940327251533, state=ENABLED in hbase:meta 2023-09-06T20:38:52,475 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16940327251533 execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:38:52,477 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-16940327251533 in 1.2200 sec 2023-09-06T20:38:53,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-09-06T20:38:53,371 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns4:test-16940327251533, procId: 22 completed 2023-09-06T20:38:53,371 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-16940327251533 get assigned. Timeout = 60000ms 2023-09-06T20:38:53,371 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-06T20:38:53,375 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-16940327251533 assigned to meta. Checking AM states. 2023-09-06T20:38:53,376 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-09-06T20:38:53,376 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-16940327251533 assigned. 2023-09-06T20:38:53,376 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x38df351f to 127.0.0.1:58035 2023-09-06T20:38:53,376 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:38:53,416 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] hbase.ResourceChecker(147): before: backup.TestFullBackupWithFailures#testFullBackupWithFailures Thread=584, OpenFileDescriptor=660, MaxFileDescriptor=60000, SystemLoadAverage=152, ProcessCount=168, AvailableMemoryMB=3643 2023-09-06T20:38:53,416 WARN [Listener at jenkins-hbase3.apache.org/41977 {}] hbase.ResourceChecker(130): Thread=584 is superior to 500 2023-09-06T20:38:53,420 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] backup.TestFullBackupWithFailures(54): Running stage 0 2023-09-06T20:38:53,436 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'backup'} 2023-09-06T20:38:53,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-09-06T20:38:53,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=25 2023-09-06T20:38:53,452 DEBUG [Listener at localhost/38777-EventThread {}] zookeeper.ZKWatcher(604): master:41705-0x100ef01325b0000, quorum=127.0.0.1:58035, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-09-06T20:38:53,456 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=25, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 19 msec 2023-09-06T20:38:53,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=25 2023-09-06T20:38:53,559 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:38:53,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-09-06T20:38:53,563 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:38:53,563 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:38:53,563 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2023-09-06T20:38:53,564 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:38:53,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-09-06T20:38:53,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-09-06T20:38:53,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-09-06T20:38:53,983 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => de261a559c0224b9046b4ead35ab8e78, NAME => 'backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0 2023-09-06T20:38:54,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-09-06T20:38:54,270 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1694032725153 2023-09-06T20:38:54,270 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1694032725153 Metrics about Tables on a single HBase RegionServer 2023-09-06T20:38:54,271 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16940327251533 2023-09-06T20:38:54,271 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-16940327251533 Metrics about Tables on a single HBase RegionServer 2023-09-06T20:38:54,272 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16940327251531 2023-09-06T20:38:54,272 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-16940327251531 Metrics about Tables on a single HBase RegionServer 2023-09-06T20:38:54,273 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16940327251532 2023-09-06T20:38:54,273 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-16940327251532 Metrics about Tables on a single HBase RegionServer 2023-09-06T20:38:54,397 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:38:54,397 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing de261a559c0224b9046b4ead35ab8e78, disabling compactions & flushes 2023-09-06T20:38:54,397 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:38:54,397 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:38:54,397 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. after waiting 0 ms 2023-09-06T20:38:54,397 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:38:54,397 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:38:54,397 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for de261a559c0224b9046b4ead35ab8e78: 2023-09-06T20:38:54,399 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:38:54,400 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1694032734399"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032734399"}]},"ts":"1694032734399"} 2023-09-06T20:38:54,402 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:38:54,403 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:38:54,403 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032734403"}]},"ts":"1694032734403"} 2023-09-06T20:38:54,405 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-09-06T20:38:54,410 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, ASSIGN}] 2023-09-06T20:38:54,412 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, ASSIGN 2023-09-06T20:38:54,413 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45559,1694032716172; forceNewPlan=false, retain=false 2023-09-06T20:38:54,565 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=de261a559c0224b9046b4ead35ab8e78, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:38:54,568 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; OpenRegionProcedure de261a559c0224b9046b4ead35ab8e78, server=jenkins-hbase3.apache.org,45559,1694032716172}] 2023-09-06T20:38:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-09-06T20:38:54,725 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(130): Open backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:38:54,726 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7254): Opening region: {ENCODED => de261a559c0224b9046b4ead35ab8e78, NAME => 'backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:38:54,726 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:38:54,727 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:38:54,727 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(888): Instantiated backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:38:54,727 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7296): checking encryption for de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:38:54,727 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7299): checking classloading for de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:38:54,729 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:38:54,732 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de261a559c0224b9046b4ead35ab8e78 columnFamilyName meta 2023-09-06T20:38:54,732 DEBUG [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:38:54,733 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.HStore(324): Store=de261a559c0224b9046b4ead35ab8e78/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:38:54,733 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:38:54,735 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de261a559c0224b9046b4ead35ab8e78 columnFamilyName session 2023-09-06T20:38:54,735 DEBUG [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:38:54,735 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.HStore(324): Store=de261a559c0224b9046b4ead35ab8e78/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:38:54,737 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:38:54,737 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:38:54,739 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-09-06T20:38:54,741 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1076): writing seq id for de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:38:54,744 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:38:54,745 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): Opened de261a559c0224b9046b4ead35ab8e78; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10817661120, jitterRate=0.007473200559616089}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-09-06T20:38:54,745 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(992): Region open journal for de261a559c0224b9046b4ead35ab8e78: 2023-09-06T20:38:54,747 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78., pid=28, masterSystemTime=1694032734720 2023-09-06T20:38:54,749 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:38:54,749 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(158): Opened backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:38:54,750 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=de261a559c0224b9046b4ead35ab8e78, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:38:54,754 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=28, resume processing ppid=27 2023-09-06T20:38:54,754 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=28, ppid=27, state=SUCCESS; OpenRegionProcedure de261a559c0224b9046b4ead35ab8e78, server=jenkins-hbase3.apache.org,45559,1694032716172 in 184 msec 2023-09-06T20:38:54,757 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=27, resume processing ppid=26 2023-09-06T20:38:54,757 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, ASSIGN in 344 msec 2023-09-06T20:38:54,759 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:38:54,759 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032734759"}]},"ts":"1694032734759"} 2023-09-06T20:38:54,761 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-09-06T20:38:54,766 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:38:54,769 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=26, state=SUCCESS; CreateTableProcedure table=backup:system in 1.2070 sec 2023-09-06T20:38:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-09-06T20:38:55,676 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: backup:system, procId: 26 completed 2023-09-06T20:38:55,690 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-09-06T20:38:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-09-06T20:38:55,694 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-09-06T20:38:55,694 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:38:55,694 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2023-09-06T20:38:55,695 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-09-06T20:38:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-09-06T20:38:55,708 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => bba17f338feef6f7f45485272a6c03ea, NAME => 'backup:system_bulk,,1694032735690.bba17f338feef6f7f45485272a6c03ea.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0 2023-09-06T20:38:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-09-06T20:38:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-09-06T20:38:56,119 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1694032735690.bba17f338feef6f7f45485272a6c03ea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:38:56,119 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing bba17f338feef6f7f45485272a6c03ea, disabling compactions & flushes 2023-09-06T20:38:56,119 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1694032735690.bba17f338feef6f7f45485272a6c03ea. 2023-09-06T20:38:56,120 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1694032735690.bba17f338feef6f7f45485272a6c03ea. 2023-09-06T20:38:56,120 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1694032735690.bba17f338feef6f7f45485272a6c03ea. after waiting 0 ms 2023-09-06T20:38:56,120 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1694032735690.bba17f338feef6f7f45485272a6c03ea. 2023-09-06T20:38:56,120 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1694032735690.bba17f338feef6f7f45485272a6c03ea. 2023-09-06T20:38:56,120 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for bba17f338feef6f7f45485272a6c03ea: 2023-09-06T20:38:56,121 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-09-06T20:38:56,121 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1694032735690.bba17f338feef6f7f45485272a6c03ea.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1694032736121"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032736121"}]},"ts":"1694032736121"} 2023-09-06T20:38:56,124 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:38:56,125 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-09-06T20:38:56,125 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032736125"}]},"ts":"1694032736125"} 2023-09-06T20:38:56,127 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-09-06T20:38:56,133 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=bba17f338feef6f7f45485272a6c03ea, ASSIGN}] 2023-09-06T20:38:56,135 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=bba17f338feef6f7f45485272a6c03ea, ASSIGN 2023-09-06T20:38:56,136 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=bba17f338feef6f7f45485272a6c03ea, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45559,1694032716172; forceNewPlan=false, retain=false 2023-09-06T20:38:56,288 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=bba17f338feef6f7f45485272a6c03ea, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:38:56,290 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE; OpenRegionProcedure bba17f338feef6f7f45485272a6c03ea, server=jenkins-hbase3.apache.org,45559,1694032716172}] 2023-09-06T20:38:56,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-09-06T20:38:56,447 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1694032735690.bba17f338feef6f7f45485272a6c03ea. 2023-09-06T20:38:56,448 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7254): Opening region: {ENCODED => bba17f338feef6f7f45485272a6c03ea, NAME => 'backup:system_bulk,,1694032735690.bba17f338feef6f7f45485272a6c03ea.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:38:56,448 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:38:56,448 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk bba17f338feef6f7f45485272a6c03ea 2023-09-06T20:38:56,448 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1694032735690.bba17f338feef6f7f45485272a6c03ea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:38:56,448 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7296): checking encryption for bba17f338feef6f7f45485272a6c03ea 2023-09-06T20:38:56,448 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7299): checking classloading for bba17f338feef6f7f45485272a6c03ea 2023-09-06T20:38:56,450 INFO [StoreOpener-bba17f338feef6f7f45485272a6c03ea-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region bba17f338feef6f7f45485272a6c03ea 2023-09-06T20:38:56,452 INFO [StoreOpener-bba17f338feef6f7f45485272a6c03ea-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bba17f338feef6f7f45485272a6c03ea columnFamilyName meta 2023-09-06T20:38:56,452 DEBUG [StoreOpener-bba17f338feef6f7f45485272a6c03ea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:38:56,453 INFO [StoreOpener-bba17f338feef6f7f45485272a6c03ea-1 {}] regionserver.HStore(324): Store=bba17f338feef6f7f45485272a6c03ea/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:38:56,453 INFO [StoreOpener-bba17f338feef6f7f45485272a6c03ea-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region bba17f338feef6f7f45485272a6c03ea 2023-09-06T20:38:56,455 INFO [StoreOpener-bba17f338feef6f7f45485272a6c03ea-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bba17f338feef6f7f45485272a6c03ea columnFamilyName session 2023-09-06T20:38:56,455 DEBUG [StoreOpener-bba17f338feef6f7f45485272a6c03ea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:38:56,455 INFO [StoreOpener-bba17f338feef6f7f45485272a6c03ea-1 {}] regionserver.HStore(324): Store=bba17f338feef6f7f45485272a6c03ea/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:38:56,456 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system_bulk/bba17f338feef6f7f45485272a6c03ea 2023-09-06T20:38:56,457 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system_bulk/bba17f338feef6f7f45485272a6c03ea 2023-09-06T20:38:56,458 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-09-06T20:38:56,460 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1076): writing seq id for bba17f338feef6f7f45485272a6c03ea 2023-09-06T20:38:56,462 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system_bulk/bba17f338feef6f7f45485272a6c03ea/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-09-06T20:38:56,463 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): Opened bba17f338feef6f7f45485272a6c03ea; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=12042717120, jitterRate=0.12156543135643005}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-09-06T20:38:56,463 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(992): Region open journal for bba17f338feef6f7f45485272a6c03ea: 2023-09-06T20:38:56,464 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system_bulk,,1694032735690.bba17f338feef6f7f45485272a6c03ea., pid=31, masterSystemTime=1694032736443 2023-09-06T20:38:56,466 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system_bulk,,1694032735690.bba17f338feef6f7f45485272a6c03ea. 2023-09-06T20:38:56,466 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1694032735690.bba17f338feef6f7f45485272a6c03ea. 2023-09-06T20:38:56,467 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=bba17f338feef6f7f45485272a6c03ea, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:38:56,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=31, resume processing ppid=30 2023-09-06T20:38:56,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=31, ppid=30, state=SUCCESS; OpenRegionProcedure bba17f338feef6f7f45485272a6c03ea, server=jenkins-hbase3.apache.org,45559,1694032716172 in 179 msec 2023-09-06T20:38:56,474 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=30, resume processing ppid=29 2023-09-06T20:38:56,474 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=30, ppid=29, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=bba17f338feef6f7f45485272a6c03ea, ASSIGN in 338 msec 2023-09-06T20:38:56,475 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-09-06T20:38:56,475 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032736475"}]},"ts":"1694032736475"} 2023-09-06T20:38:56,477 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-09-06T20:38:56,480 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-09-06T20:38:56,481 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=29, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 790 msec 2023-09-06T20:38:56,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-09-06T20:38:56,807 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: backup:system_bulk, procId: 29 completed 2023-09-06T20:38:56,856 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-09-06T20:38:56,856 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-09-06T20:38:56,856 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f27cfad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:38:56,856 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-16940327251533' 2023-09-06T20:38:56,857 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:38:56,859 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:57680, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:38:56,864 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:38:56,866 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48894, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:38:56,933 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:38:56,993 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:38:56,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032736993 (current time:1694032736993). 2023-09-06T20:38:56,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:38:56,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:38:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79592f10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:38:57,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:38:57,008 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48902, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:38:57,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:38:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:38:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:38:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-09-06T20:38:57,026 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:38:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-09-06T20:38:57,032 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:38:57,035 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:38:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-09-06T20:38:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-09-06T20:38:57,446 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:38:57,449 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure de261a559c0224b9046b4ead35ab8e78}] 2023-09-06T20:38:57,455 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:38:57,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45559 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2023-09-06T20:38:57,618 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:38:57,620 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2819): Flushing de261a559c0224b9046b4ead35ab8e78 2/2 column families, dataSize=45 B heapSize=632 B 2023-09-06T20:38:57,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-09-06T20:38:57,696 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/8e94389278694c96b296c117230449dd is 49, key is activesession:/session:c/1694032736972/Put/seqid=0 2023-09-06T20:38:57,921 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:38:58,116 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/8e94389278694c96b296c117230449dd 2023-09-06T20:38:58,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-09-06T20:38:58,206 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/8e94389278694c96b296c117230449dd as hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8e94389278694c96b296c117230449dd 2023-09-06T20:38:58,215 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8e94389278694c96b296c117230449dd, entries=1, sequenceid=5, filesize=4.9 K 2023-09-06T20:38:58,219 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for de261a559c0224b9046b4ead35ab8e78 in 599ms, sequenceid=5, compaction requested=false 2023-09-06T20:38:58,221 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2520): Flush status journal for de261a559c0224b9046b4ead35ab8e78: 2023-09-06T20:38:58,221 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. for snapshot_backup_system completed. 2023-09-06T20:38:58,222 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:38:58,229 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:38:58,233 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-09-06T20:38:58,233 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8e94389278694c96b296c117230449dd] hfiles 2023-09-06T20:38:58,233 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8e94389278694c96b296c117230449dd for snapshot=snapshot_backup_system 2023-09-06T20:38:58,248 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:38:58,252 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=33 2023-09-06T20:38:58,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(3995): Remote procedure done, pid=33 2023-09-06T20:38:58,255 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:38:58,256 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:38:58,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=33, resume processing ppid=32 2023-09-06T20:38:58,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=33, ppid=32, state=SUCCESS; SnapshotRegionProcedure de261a559c0224b9046b4ead35ab8e78 in 808 msec 2023-09-06T20:38:58,260 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:38:58,262 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:38:58,265 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:38:58,265 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:38:58,267 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:38:58,693 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:38:58,705 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:38:58,706 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:38:58,708 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:38:58,708 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-09-06T20:38:58,710 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=32, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.6890 sec 2023-09-06T20:38:59,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-09-06T20:38:59,138 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(115): Backup backup_1694032736907 started at 1694032739138. 2023-09-06T20:38:59,161 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(119): Backup session backup_1694032736907 has been started. 2023-09-06T20:38:59,161 ERROR [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(228): Unexpected BackupException : Failed stage stage_0 in testing java.io.IOException: Failed stage stage_0 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:188) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.runBackupAndFailAtStage(TestFullBackupWithFailures.java:67) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.testFullBackupWithFailures(TestFullBackupWithFailures.java:55) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:38:59,161 ERROR [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(240): BackupId=backup_1694032736907,startts=1694032739138,failedts=1694032739161,failedphase=REQUEST,failedmessage=Failed stage stage_0 in testing 2023-09-06T20:38:59,161 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-09-06T20:38:59,163 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-09-06T20:38:59,184 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable backup:system 2023-09-06T20:38:59,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=34, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-09-06T20:38:59,196 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032739196"}]},"ts":"1694032739196"} 2023-09-06T20:38:59,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:38:59,198 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-09-06T20:38:59,200 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-09-06T20:38:59,201 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, UNASSIGN}] 2023-09-06T20:38:59,203 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, UNASSIGN 2023-09-06T20:38:59,204 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=35 updating hbase:meta row=de261a559c0224b9046b4ead35ab8e78, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:38:59,206 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE; CloseRegionProcedure de261a559c0224b9046b4ead35ab8e78, server=jenkins-hbase3.apache.org,45559,1694032716172}] 2023-09-06T20:38:59,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:38:59,366 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(119): Close de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:38:59,366 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1668): Closing de261a559c0224b9046b4ead35ab8e78, disabling compactions & flushes 2023-09-06T20:38:59,366 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1690): Closing region backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:38:59,366 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:38:59,366 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. after waiting 0 ms 2023-09-06T20:38:59,366 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:38:59,366 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(2819): Flushing de261a559c0224b9046b4ead35ab8e78 2/2 column families, dataSize=214 B heapSize=800 B 2023-09-06T20:38:59,377 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/387d99cb22a044ee903a21485df3a670 is 218, key is session:backup_1694032736907/session:context/1694032739159/Put/seqid=0 2023-09-06T20:38:59,386 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=214 B at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/387d99cb22a044ee903a21485df3a670 2023-09-06T20:38:59,397 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/387d99cb22a044ee903a21485df3a670 as hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/387d99cb22a044ee903a21485df3a670 2023-09-06T20:38:59,404 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/387d99cb22a044ee903a21485df3a670, entries=1, sequenceid=9, filesize=5.2 K 2023-09-06T20:38:59,407 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(3022): Finished flush of dataSize ~214 B/214, heapSize ~528 B/528, currentSize=0 B/0 for de261a559c0224b9046b4ead35ab8e78 in 41ms, sequenceid=9, compaction requested=false 2023-09-06T20:38:59,416 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2023-09-06T20:38:59,419 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:38:59,420 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1909): Closed backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:38:59,420 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1622): Region close journal for de261a559c0224b9046b4ead35ab8e78: 2023-09-06T20:38:59,422 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(163): Closed de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:38:59,423 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=35 updating hbase:meta row=de261a559c0224b9046b4ead35ab8e78, regionState=CLOSED 2023-09-06T20:38:59,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=36, resume processing ppid=35 2023-09-06T20:38:59,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=36, ppid=35, state=SUCCESS; CloseRegionProcedure de261a559c0224b9046b4ead35ab8e78, server=jenkins-hbase3.apache.org,45559,1694032716172 in 218 msec 2023-09-06T20:38:59,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=35, resume processing ppid=34 2023-09-06T20:38:59,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=35, ppid=34, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, UNASSIGN in 225 msec 2023-09-06T20:38:59,429 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032739429"}]},"ts":"1694032739429"} 2023-09-06T20:38:59,431 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-09-06T20:38:59,432 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-09-06T20:38:59,434 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=34, state=SUCCESS; DisableTableProcedure table=backup:system in 247 msec 2023-09-06T20:38:59,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-09-06T20:38:59,511 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1694032739511 table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:38:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032739511 (current time:1694032739511). 2023-09-06T20:38:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:38:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1694032739511 VERSION not specified, setting to 2 2023-09-06T20:38:59,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68adac34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:38:59,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:38:59,521 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48908, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:38:59,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:38:59,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:38:59,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=37, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032739511 table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:38:59,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032739511 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 37 2023-09-06T20:38:59,526 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032739511 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:38:59,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=37 2023-09-06T20:38:59,527 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032739511 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:38:59,529 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032739511 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:38:59,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=37 2023-09-06T20:38:59,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=37 2023-09-06T20:38:59,941 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032739511 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-09-06T20:38:59,942 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-09-06T20:38:59,942 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-09-06T20:38:59,943 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:38:59,943 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(327): No files under family: meta 2023-09-06T20:38:59,943 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:38:59,945 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/387d99cb22a044ee903a21485df3a670, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8e94389278694c96b296c117230449dd] hfiles 2023-09-06T20:38:59,945 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/2): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/387d99cb22a044ee903a21485df3a670 2023-09-06T20:38:59,945 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/2): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8e94389278694c96b296c117230449dd 2023-09-06T20:39:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=37 2023-09-06T20:39:00,356 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => de261a559c0224b9046b4ead35ab8e78, NAME => 'backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-09-06T20:39:00,358 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032739511 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:39:00,359 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032739511 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:39:00,359 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1694032739511 2023-09-06T20:39:00,361 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1694032739511 2023-09-06T20:39:00,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=37 2023-09-06T20:39:00,792 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032739511 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:39:00,804 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032739511 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:39:00,804 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1694032739511 to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1694032739511 2023-09-06T20:39:00,806 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032739511 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:39:00,806 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032739511 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 37 2023-09-06T20:39:00,808 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=37, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032739511 table=backup:system type=FLUSH ttl=0 } in 1.2830 sec 2023-09-06T20:39:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=37 2023-09-06T20:39:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=38, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032736993 type: FLUSH version: 2 ttl: 0 ) 2023-09-06T20:39:01,662 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(1015): Restore snapshot=snapshot_backup_system as table=backup:system 2023-09-06T20:39:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-09-06T20:39:01,690 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/.tabledesc/.tableinfo.0000000001.635 2023-09-06T20:39:01,690 INFO [PEWorker-3 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2023-09-06T20:39:01,692 INFO [PEWorker-3 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:39:01,699 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032736993 type: FLUSH version: 2 ttl: 0 2023-09-06T20:39:01,700 DEBUG [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system 2023-09-06T20:39:01,704 DEBUG [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-09-06T20:39:01,704 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(227): region to restore: de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:01,713 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:39:01,732 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/387d99cb22a044ee903a21485df3a670 to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/archive/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/387d99cb22a044ee903a21485df3a670 2023-09-06T20:39:01,733 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:39:01,733 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032736993 type: FLUSH version: 2 ttl: 0 2023-09-06T20:39:01,735 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1694032741735"}]},"ts":"9223372036854775807"} 2023-09-06T20:39:01,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-09-06T20:39:01,795 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-06T20:39:01,795 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => de261a559c0224b9046b4ead35ab8e78, NAME => 'backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:39:01,795 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1694032741736"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032741736"}]},"ts":"1694032741736"} 2023-09-06T20:39:01,797 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:39:01,797 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-09-06T20:39:01,797 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => de261a559c0224b9046b4ead35ab8e78, NAME => 'backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:39:01,798 INFO [PEWorker-3 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-09-06T20:39:01,801 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=38, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032736993 type: FLUSH version: 2 ttl: 0 ) in 149 msec 2023-09-06T20:39:01,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-09-06T20:39:01,978 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1694032739511" 2023-09-06T20:39:01,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(378): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1694032739511 2023-09-06T20:39:01,990 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$11(2631): Client=jenkins//172.31.12.81 enable backup:system 2023-09-06T20:39:01,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=39, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-09-06T20:39:02,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-09-06T20:39:02,003 INFO [PEWorker-1 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-09-06T20:39:02,003 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032742003"}]},"ts":"1694032742003"} 2023-09-06T20:39:02,005 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-09-06T20:39:02,008 DEBUG [PEWorker-1 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-09-06T20:39:02,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, ASSIGN}] 2023-09-06T20:39:02,009 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, ASSIGN 2023-09-06T20:39:02,010 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-09-06T20:39:02,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-09-06T20:39:02,162 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=de261a559c0224b9046b4ead35ab8e78, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:02,164 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE; OpenRegionProcedure de261a559c0224b9046b4ead35ab8e78, server=jenkins-hbase3.apache.org,45559,1694032716172}] 2023-09-06T20:39:02,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-09-06T20:39:02,321 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(130): Open backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:02,322 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7254): Opening region: {ENCODED => de261a559c0224b9046b4ead35ab8e78, NAME => 'backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:39:02,322 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:39:02,322 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:02,322 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(888): Instantiated backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:39:02,323 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7296): checking encryption for de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:02,323 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7299): checking classloading for de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:02,325 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:02,326 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de261a559c0224b9046b4ead35ab8e78 columnFamilyName meta 2023-09-06T20:39:02,326 DEBUG [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:39:02,327 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.HStore(324): Store=de261a559c0224b9046b4ead35ab8e78/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:39:02,327 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:02,346 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de261a559c0224b9046b4ead35ab8e78 columnFamilyName session 2023-09-06T20:39:02,347 DEBUG [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:39:02,361 DEBUG [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8e94389278694c96b296c117230449dd 2023-09-06T20:39:02,361 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.HStore(324): Store=de261a559c0224b9046b4ead35ab8e78/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:39:02,363 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:02,367 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:02,369 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-09-06T20:39:02,371 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1076): writing seq id for de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:02,373 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): Opened de261a559c0224b9046b4ead35ab8e78; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11697845120, jitterRate=0.08944672346115112}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-09-06T20:39:02,373 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(992): Region open journal for de261a559c0224b9046b4ead35ab8e78: 2023-09-06T20:39:02,375 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78., pid=41, masterSystemTime=1694032742316 2023-09-06T20:39:02,377 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:02,377 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(158): Opened backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:02,377 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=de261a559c0224b9046b4ead35ab8e78, regionState=OPEN, openSeqNum=13, regionLocation=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:02,382 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=41, resume processing ppid=40 2023-09-06T20:39:02,382 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=41, ppid=40, state=SUCCESS; OpenRegionProcedure de261a559c0224b9046b4ead35ab8e78, server=jenkins-hbase3.apache.org,45559,1694032716172 in 216 msec 2023-09-06T20:39:02,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=40, resume processing ppid=39 2023-09-06T20:39:02,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=40, ppid=39, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, ASSIGN in 374 msec 2023-09-06T20:39:02,385 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032742385"}]},"ts":"1694032742385"} 2023-09-06T20:39:02,386 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-09-06T20:39:02,388 INFO [PEWorker-2 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-09-06T20:39:02,391 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=39, state=SUCCESS; EnableTableProcedure table=backup:system in 397 msec 2023-09-06T20:39:02,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-09-06T20:39:02,608 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(1525): Done restoring backup system table 2023-09-06T20:39:02,608 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:39:02,616 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:39:02,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:39:02,620 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:39:02,621 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-09-06T20:39:02,623 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:39:02,646 ERROR [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(246): Backup backup_1694032736907 failed. Backup session finished. Status: FAILURE 2023-09-06T20:39:02,646 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:39:02,646 ERROR [Listener at jenkins-hbase3.apache.org/41977 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_0 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:255) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.runBackupAndFailAtStage(TestFullBackupWithFailures.java:67) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.testFullBackupWithFailures(TestFullBackupWithFailures.java:55) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_0 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:188) ~[test-classes/:?] ... 32 more 2023-09-06T20:39:02,670 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] backup.TestFullBackupWithFailures(54): Running stage 1 2023-09-06T20:39:02,700 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40373818, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:39:02,701 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:39:02,703 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50828, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:39:02,708 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:39:02,711 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:56720, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:39:02,773 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:39:02,778 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:39:02,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032742778 (current time:1694032742778). 2023-09-06T20:39:02,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:39:02,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:39:02,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dab9e12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:39:02,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:39:02,790 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:56724, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:39:02,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:39:02,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:39:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=42, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:39:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 42 2023-09-06T20:39:02,795 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:39:02,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-09-06T20:39:02,796 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:39:02,799 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:39:02,813 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:39:02,813 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure de261a559c0224b9046b4ead35ab8e78}] 2023-09-06T20:39:02,815 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:02,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-09-06T20:39:02,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45559 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=43 2023-09-06T20:39:02,967 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:02,968 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(2819): Flushing de261a559c0224b9046b4ead35ab8e78 2/2 column families, dataSize=341 B heapSize=1.05 KB 2023-09-06T20:39:02,985 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/1e63938d58d1473a8b0b4e3f75f5e1eb is 256, key is session:backup_1694032736907/session:context/1694032742622/Put/seqid=0 2023-09-06T20:39:03,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-09-06T20:39:03,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-09-06T20:39:03,423 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=341 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/1e63938d58d1473a8b0b4e3f75f5e1eb 2023-09-06T20:39:03,432 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/1e63938d58d1473a8b0b4e3f75f5e1eb as hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/1e63938d58d1473a8b0b4e3f75f5e1eb 2023-09-06T20:39:03,439 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/1e63938d58d1473a8b0b4e3f75f5e1eb, entries=2, sequenceid=18, filesize=5.2 K 2023-09-06T20:39:03,441 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(3022): Finished flush of dataSize ~341 B/341, heapSize ~808 B/808, currentSize=0 B/0 for de261a559c0224b9046b4ead35ab8e78 in 473ms, sequenceid=18, compaction requested=false 2023-09-06T20:39:03,441 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(2520): Flush status journal for de261a559c0224b9046b4ead35ab8e78: 2023-09-06T20:39:03,441 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. for snapshot_backup_system completed. 2023-09-06T20:39:03,441 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:39:03,441 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:39:03,442 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-09-06T20:39:03,442 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8e94389278694c96b296c117230449dd, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/1e63938d58d1473a8b0b4e3f75f5e1eb] hfiles 2023-09-06T20:39:03,442 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8e94389278694c96b296c117230449dd for snapshot=snapshot_backup_system 2023-09-06T20:39:03,442 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/1e63938d58d1473a8b0b4e3f75f5e1eb for snapshot=snapshot_backup_system 2023-09-06T20:39:03,850 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:03,851 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=43 2023-09-06T20:39:03,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(3995): Remote procedure done, pid=43 2023-09-06T20:39:03,851 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:03,853 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:03,856 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=43, resume processing ppid=42 2023-09-06T20:39:03,856 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:39:03,856 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=43, ppid=42, state=SUCCESS; SnapshotRegionProcedure de261a559c0224b9046b4ead35ab8e78 in 1.0410 sec 2023-09-06T20:39:03,857 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:39:03,858 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:39:03,858 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:39:03,859 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:39:03,882 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:39:03,891 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:39:03,891 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:39:03,893 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:39:03,894 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 42 2023-09-06T20:39:03,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=42, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.1020 sec 2023-09-06T20:39:03,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-09-06T20:39:03,902 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(115): Backup backup_1694032742753 started at 1694032743902. 2023-09-06T20:39:03,904 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(119): Backup session backup_1694032742753 has been started. 2023-09-06T20:39:03,907 ERROR [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(228): Unexpected BackupException : Failed stage stage_1 in testing java.io.IOException: Failed stage stage_1 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:199) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.runBackupAndFailAtStage(TestFullBackupWithFailures.java:67) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.testFullBackupWithFailures(TestFullBackupWithFailures.java:55) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:39:03,908 ERROR [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(240): BackupId=backup_1694032742753,startts=1694032743902,failedts=1694032743908,failedphase=REQUEST,failedmessage=Failed stage stage_1 in testing 2023-09-06T20:39:03,908 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-09-06T20:39:03,909 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-09-06T20:39:03,918 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable backup:system 2023-09-06T20:39:03,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=44, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-09-06T20:39:03,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-09-06T20:39:03,923 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032743923"}]},"ts":"1694032743923"} 2023-09-06T20:39:03,924 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-09-06T20:39:03,926 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-09-06T20:39:03,927 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, UNASSIGN}] 2023-09-06T20:39:03,928 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, UNASSIGN 2023-09-06T20:39:03,929 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=45 updating hbase:meta row=de261a559c0224b9046b4ead35ab8e78, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:03,931 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE; CloseRegionProcedure de261a559c0224b9046b4ead35ab8e78, server=jenkins-hbase3.apache.org,45559,1694032716172}] 2023-09-06T20:39:04,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-09-06T20:39:04,082 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] handler.UnassignRegionHandler(119): Close de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:04,083 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1668): Closing de261a559c0224b9046b4ead35ab8e78, disabling compactions & flushes 2023-09-06T20:39:04,083 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1690): Closing region backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:04,083 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:04,083 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. after waiting 0 ms 2023-09-06T20:39:04,083 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:04,083 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(2819): Flushing de261a559c0224b9046b4ead35ab8e78 2/2 column families, dataSize=289 B heapSize=952 B 2023-09-06T20:39:04,098 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/meta/8b53004f36d44437b01556cab4a624c9 is 79, key is startcode:hdfs://localhost:34451/backupUT/meta:startcode/1694032743906/Put/seqid=0 2023-09-06T20:39:04,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-09-06T20:39:04,270 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-09-06T20:39:04,270 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-09-06T20:39:04,271 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-09-06T20:39:04,271 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-09-06T20:39:04,513 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=75 B at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/meta/8b53004f36d44437b01556cab4a624c9 2023-09-06T20:39:04,525 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/df2a46eb4422477195ce163fc6d95176 is 218, key is session:backup_1694032742753/session:context/1694032743903/Put/seqid=0 2023-09-06T20:39:04,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-09-06T20:39:04,923 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:39:04,934 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=214 B at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/df2a46eb4422477195ce163fc6d95176 2023-09-06T20:39:04,942 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/meta/8b53004f36d44437b01556cab4a624c9 as hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/meta/8b53004f36d44437b01556cab4a624c9 2023-09-06T20:39:04,950 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/meta/8b53004f36d44437b01556cab4a624c9, entries=1, sequenceid=23, filesize=5.1 K 2023-09-06T20:39:04,951 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/df2a46eb4422477195ce163fc6d95176 as hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/df2a46eb4422477195ce163fc6d95176 2023-09-06T20:39:04,959 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/df2a46eb4422477195ce163fc6d95176, entries=1, sequenceid=23, filesize=5.2 K 2023-09-06T20:39:04,960 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(3022): Finished flush of dataSize ~289 B/289, heapSize ~920 B/920, currentSize=0 B/0 for de261a559c0224b9046b4ead35ab8e78 in 877ms, sequenceid=23, compaction requested=true 2023-09-06T20:39:04,977 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=12 2023-09-06T20:39:04,978 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:39:04,979 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1909): Closed backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:04,979 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1622): Region close journal for de261a559c0224b9046b4ead35ab8e78: 2023-09-06T20:39:04,981 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] handler.UnassignRegionHandler(163): Closed de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:04,982 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=45 updating hbase:meta row=de261a559c0224b9046b4ead35ab8e78, regionState=CLOSED 2023-09-06T20:39:04,986 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=46, resume processing ppid=45 2023-09-06T20:39:04,986 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=46, ppid=45, state=SUCCESS; CloseRegionProcedure de261a559c0224b9046b4ead35ab8e78, server=jenkins-hbase3.apache.org,45559,1694032716172 in 1.0530 sec 2023-09-06T20:39:04,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=45, resume processing ppid=44 2023-09-06T20:39:04,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=45, ppid=44, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, UNASSIGN in 1.0590 sec 2023-09-06T20:39:04,989 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032744988"}]},"ts":"1694032744988"} 2023-09-06T20:39:04,990 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-09-06T20:39:04,992 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-09-06T20:39:04,994 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=44, state=SUCCESS; DisableTableProcedure table=backup:system in 1.0750 sec 2023-09-06T20:39:05,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-09-06T20:39:05,038 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1694032745037 table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:39:05,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032745038 (current time:1694032745038). 2023-09-06T20:39:05,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:39:05,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1694032745037 VERSION not specified, setting to 2 2023-09-06T20:39:05,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65390592, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:39:05,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:39:05,049 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:56726, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:39:05,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:39:05,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:39:05,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=47, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032745037 table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:39:05,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032745037 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2023-09-06T20:39:05,053 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032745037 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:39:05,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-09-06T20:39:05,055 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032745037 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:39:05,057 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032745037 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:39:05,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-09-06T20:39:05,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-09-06T20:39:05,480 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032745037 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-09-06T20:39:05,481 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-09-06T20:39:05,481 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-09-06T20:39:05,482 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:39:05,483 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/meta/8b53004f36d44437b01556cab4a624c9] hfiles 2023-09-06T20:39:05,483 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/1): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/meta/8b53004f36d44437b01556cab4a624c9 2023-09-06T20:39:05,483 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:39:05,485 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/1e63938d58d1473a8b0b4e3f75f5e1eb, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8e94389278694c96b296c117230449dd, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/df2a46eb4422477195ce163fc6d95176] hfiles 2023-09-06T20:39:05,485 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/3): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/1e63938d58d1473a8b0b4e3f75f5e1eb 2023-09-06T20:39:05,485 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/3): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8e94389278694c96b296c117230449dd 2023-09-06T20:39:05,485 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/3): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/df2a46eb4422477195ce163fc6d95176 2023-09-06T20:39:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-09-06T20:39:05,904 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => de261a559c0224b9046b4ead35ab8e78, NAME => 'backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-09-06T20:39:05,905 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032745037 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:39:05,906 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032745037 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:39:05,906 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1694032745037 2023-09-06T20:39:05,907 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1694032745037 2023-09-06T20:39:05,916 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032745037 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:39:05,924 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032745037 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:39:05,925 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1694032745037 to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1694032745037 2023-09-06T20:39:05,926 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032745037 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:39:05,926 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032745037 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2023-09-06T20:39:05,927 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=47, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032745037 table=backup:system type=FLUSH ttl=0 } in 875 msec 2023-09-06T20:39:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-09-06T20:39:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=48, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032742778 type: FLUSH version: 2 ttl: 0 ) 2023-09-06T20:39:06,170 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(1015): Restore snapshot=snapshot_backup_system as table=backup:system 2023-09-06T20:39:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-06T20:39:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-06T20:39:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-06T20:39:06,592 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2023-09-06T20:39:06,592 INFO [PEWorker-5 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/.tabledesc/.tableinfo.0000000003.635 2023-09-06T20:39:06,593 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:39:06,596 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032742778 type: FLUSH version: 2 ttl: 0 2023-09-06T20:39:06,596 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system 2023-09-06T20:39:06,600 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-09-06T20:39:06,600 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(227): region to restore: de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:06,609 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/meta/8b53004f36d44437b01556cab4a624c9 to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/archive/data/backup/system/de261a559c0224b9046b4ead35ab8e78/meta/8b53004f36d44437b01556cab4a624c9 2023-09-06T20:39:06,609 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:39:06,614 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/df2a46eb4422477195ce163fc6d95176 to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/archive/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/df2a46eb4422477195ce163fc6d95176 2023-09-06T20:39:06,614 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:39:06,615 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032742778 type: FLUSH version: 2 ttl: 0 2023-09-06T20:39:06,616 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1694032746616"}]},"ts":"9223372036854775807"} 2023-09-06T20:39:06,618 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-06T20:39:06,618 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => de261a559c0224b9046b4ead35ab8e78, NAME => 'backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:39:06,619 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1694032746617"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032746617"}]},"ts":"1694032746617"} 2023-09-06T20:39:06,620 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:39:06,620 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-09-06T20:39:06,620 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => de261a559c0224b9046b4ead35ab8e78, NAME => 'backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:39:06,620 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-09-06T20:39:06,623 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=48, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032742778 type: FLUSH version: 2 ttl: 0 ) in 452 msec 2023-09-06T20:39:06,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-09-06T20:39:06,778 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1694032745037" 2023-09-06T20:39:06,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(378): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1694032745037 2023-09-06T20:39:06,784 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$11(2631): Client=jenkins//172.31.12.81 enable backup:system 2023-09-06T20:39:06,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=49, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-09-06T20:39:06,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=49 2023-09-06T20:39:06,788 INFO [PEWorker-3 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-09-06T20:39:06,788 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032746788"}]},"ts":"1694032746788"} 2023-09-06T20:39:06,790 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-09-06T20:39:06,792 DEBUG [PEWorker-3 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-09-06T20:39:06,793 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, ASSIGN}] 2023-09-06T20:39:06,795 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, ASSIGN 2023-09-06T20:39:06,795 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-09-06T20:39:06,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=49 2023-09-06T20:39:06,947 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=50 updating hbase:meta row=de261a559c0224b9046b4ead35ab8e78, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:06,949 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE; OpenRegionProcedure de261a559c0224b9046b4ead35ab8e78, server=jenkins-hbase3.apache.org,45559,1694032716172}] 2023-09-06T20:39:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=49 2023-09-06T20:39:07,105 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(130): Open backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:07,106 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7254): Opening region: {ENCODED => de261a559c0224b9046b4ead35ab8e78, NAME => 'backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:39:07,106 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:39:07,106 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:07,107 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(888): Instantiated backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:39:07,107 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7296): checking encryption for de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:07,107 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7299): checking classloading for de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:07,108 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:07,110 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de261a559c0224b9046b4ead35ab8e78 columnFamilyName meta 2023-09-06T20:39:07,110 DEBUG [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:39:07,110 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.HStore(324): Store=de261a559c0224b9046b4ead35ab8e78/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:39:07,110 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:07,111 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de261a559c0224b9046b4ead35ab8e78 columnFamilyName session 2023-09-06T20:39:07,112 DEBUG [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:39:07,119 DEBUG [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/1e63938d58d1473a8b0b4e3f75f5e1eb 2023-09-06T20:39:07,125 DEBUG [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8e94389278694c96b296c117230449dd 2023-09-06T20:39:07,125 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.HStore(324): Store=de261a559c0224b9046b4ead35ab8e78/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:39:07,126 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:07,128 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:07,130 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-09-06T20:39:07,131 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1076): writing seq id for de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:07,132 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): Opened de261a559c0224b9046b4ead35ab8e78; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11167120640, jitterRate=0.04001915454864502}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-09-06T20:39:07,133 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(992): Region open journal for de261a559c0224b9046b4ead35ab8e78: 2023-09-06T20:39:07,134 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78., pid=51, masterSystemTime=1694032747101 2023-09-06T20:39:07,136 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:07,136 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(158): Opened backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:07,136 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=50 updating hbase:meta row=de261a559c0224b9046b4ead35ab8e78, regionState=OPEN, openSeqNum=27, regionLocation=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:07,140 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=51, resume processing ppid=50 2023-09-06T20:39:07,140 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=51, ppid=50, state=SUCCESS; OpenRegionProcedure de261a559c0224b9046b4ead35ab8e78, server=jenkins-hbase3.apache.org,45559,1694032716172 in 189 msec 2023-09-06T20:39:07,142 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=50, resume processing ppid=49 2023-09-06T20:39:07,142 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=50, ppid=49, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, ASSIGN in 347 msec 2023-09-06T20:39:07,143 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032747143"}]},"ts":"1694032747143"} 2023-09-06T20:39:07,144 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-09-06T20:39:07,146 INFO [PEWorker-1 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-09-06T20:39:07,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=49, state=SUCCESS; EnableTableProcedure table=backup:system in 363 msec 2023-09-06T20:39:07,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=49 2023-09-06T20:39:07,392 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(1525): Done restoring backup system table 2023-09-06T20:39:07,392 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:39:07,398 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:39:07,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:39:07,402 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:39:07,402 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-09-06T20:39:07,404 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:39:07,410 ERROR [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(246): Backup backup_1694032742753 failed. Backup session finished. Status: FAILURE 2023-09-06T20:39:07,410 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:39:07,410 ERROR [Listener at jenkins-hbase3.apache.org/41977 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_1 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:255) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.runBackupAndFailAtStage(TestFullBackupWithFailures.java:67) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.testFullBackupWithFailures(TestFullBackupWithFailures.java:55) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_1 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:199) ~[test-classes/:?] ... 32 more 2023-09-06T20:39:07,445 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] backup.TestFullBackupWithFailures(54): Running stage 2 2023-09-06T20:39:07,467 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23caada4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:39:07,468 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:39:07,470 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50842, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:39:07,474 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:39:07,475 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:56732, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:39:07,520 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:39:07,525 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:39:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032747525 (current time:1694032747525). 2023-09-06T20:39:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:39:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:39:07,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73c7ae40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:39:07,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:39:07,536 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:56744, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:39:07,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:39:07,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:39:07,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=52, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:39:07,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-09-06T20:39:07,541 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:39:07,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-09-06T20:39:07,542 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:39:07,544 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:39:07,552 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:39:07,552 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure de261a559c0224b9046b4ead35ab8e78}] 2023-09-06T20:39:07,554 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:07,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-09-06T20:39:07,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45559 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=53 2023-09-06T20:39:07,706 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:07,707 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2819): Flushing de261a559c0224b9046b4ead35ab8e78 2/2 column families, dataSize=341 B heapSize=1.05 KB 2023-09-06T20:39:07,722 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/8382c9eb13b242258a13aab290003d1c is 256, key is session:backup_1694032742753/session:context/1694032747403/Put/seqid=0 2023-09-06T20:39:07,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-09-06T20:39:08,128 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=341 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/8382c9eb13b242258a13aab290003d1c 2023-09-06T20:39:08,137 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/8382c9eb13b242258a13aab290003d1c as hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8382c9eb13b242258a13aab290003d1c 2023-09-06T20:39:08,144 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8382c9eb13b242258a13aab290003d1c, entries=2, sequenceid=32, filesize=5.2 K 2023-09-06T20:39:08,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-09-06T20:39:08,145 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(3022): Finished flush of dataSize ~341 B/341, heapSize ~808 B/808, currentSize=0 B/0 for de261a559c0224b9046b4ead35ab8e78 in 438ms, sequenceid=32, compaction requested=true 2023-09-06T20:39:08,146 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2520): Flush status journal for de261a559c0224b9046b4ead35ab8e78: 2023-09-06T20:39:08,146 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. for snapshot_backup_system completed. 2023-09-06T20:39:08,146 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:39:08,146 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:39:08,146 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-09-06T20:39:08,146 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8e94389278694c96b296c117230449dd, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/1e63938d58d1473a8b0b4e3f75f5e1eb, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8382c9eb13b242258a13aab290003d1c] hfiles 2023-09-06T20:39:08,146 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8e94389278694c96b296c117230449dd for snapshot=snapshot_backup_system 2023-09-06T20:39:08,146 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/1e63938d58d1473a8b0b4e3f75f5e1eb for snapshot=snapshot_backup_system 2023-09-06T20:39:08,146 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8382c9eb13b242258a13aab290003d1c for snapshot=snapshot_backup_system 2023-09-06T20:39:08,153 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:08,153 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=53 2023-09-06T20:39:08,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(3995): Remote procedure done, pid=53 2023-09-06T20:39:08,154 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:08,155 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:08,157 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=53, resume processing ppid=52 2023-09-06T20:39:08,158 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=53, ppid=52, state=SUCCESS; SnapshotRegionProcedure de261a559c0224b9046b4ead35ab8e78 in 604 msec 2023-09-06T20:39:08,157 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:39:08,158 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:39:08,159 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:39:08,159 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:39:08,160 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:39:08,175 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:39:08,182 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:39:08,183 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:39:08,184 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:39:08,184 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-09-06T20:39:08,185 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=52, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 646 msec 2023-09-06T20:39:08,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-09-06T20:39:08,647 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(115): Backup backup_1694032747504 started at 1694032748647. 2023-09-06T20:39:08,649 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(119): Backup session backup_1694032747504 has been started. 2023-09-06T20:39:08,652 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] backup.TestBackupBase$FullTableBackupClientForTest(205): Execute roll log procedure for full backup ... 2023-09-06T20:39:08,663 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-09-06T20:39:08,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-09-06T20:39:08,672 INFO [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-09-06T20:39:08,672 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-09-06T20:39:08,673 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-09-06T20:39:08,673 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-09-06T20:39:08,673 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:39:08,675 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:08,675 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:39:08,675 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:39:08,675 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-09-06T20:39:08,675 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-09-06T20:39:08,676 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-09-06T20:39:08,676 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:39:08,676 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-09-06T20:39:08,678 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-09-06T20:39:08,681 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-09-06T20:39:08,682 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-09-06T20:39:08,682 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-09-06T20:39:08,683 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-09-06T20:39:08,683 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-09-06T20:39:08,683 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,45559,1694032716172' joining acquired barrier for procedure (rolllog) in zk 2023-09-06T20:39:08,686 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-09-06T20:39:08,686 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:08,686 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:39:08,687 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:39:08,687 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-09-06T20:39:08,687 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:39:08,687 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:39:08,688 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:39:08,688 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:08,689 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:39:08,689 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,45559,1694032716172' joining acquired barrier for procedure 'rolllog' on coordinator 2023-09-06T20:39:08,689 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@717709aa[Count = 0] remaining members to acquire global barrier 2023-09-06T20:39:08,689 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-09-06T20:39:08,689 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-09-06T20:39:08,691 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-09-06T20:39:08,692 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-09-06T20:39:08,692 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-09-06T20:39:08,692 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-09-06T20:39:08,693 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-09-06T20:39:08,693 DEBUG [rs(jenkins-hbase3.apache.org,45559,1694032716172)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:08,693 INFO [rs(jenkins-hbase3.apache.org,45559,1694032716172)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1694032718597 highest: 1694032718597 on jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:08,693 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45559%2C1694032716172.meta:.meta(num 1694032719123) roll requested 2023-09-06T20:39:08,704 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.meta.1694032748693.meta, exclude list is [], retry=0 2023-09-06T20:39:08,709 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41237,DS-07aa54ab-1cb2-4831-8bde-a715b49e632a,DISK] 2023-09-06T20:39:08,712 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.meta.1694032719123.meta with entries=56, filesize=15.96 KB; new WAL /user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.meta.1694032748693.meta 2023-09-06T20:39:08,713 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41237,DS-07aa54ab-1cb2-4831-8bde-a715b49e632a,DISK]] 2023-09-06T20:39:08,713 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.meta.1694032719123.meta is not closed yet, will try archiving it next time 2023-09-06T20:39:08,713 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45559%2C1694032716172:(num 1694032718597) roll requested 2023-09-06T20:39:08,728 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.1694032748713, exclude list is [], retry=0 2023-09-06T20:39:08,732 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41237,DS-07aa54ab-1cb2-4831-8bde-a715b49e632a,DISK] 2023-09-06T20:39:08,735 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.1694032718597 with entries=40, filesize=9.29 KB; new WAL /user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.1694032748713 2023-09-06T20:39:08,735 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41237,DS-07aa54ab-1cb2-4831-8bde-a715b49e632a,DISK]] 2023-09-06T20:39:08,735 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.1694032718597 is not closed yet, will try archiving it next time 2023-09-06T20:39:08,754 DEBUG [rs(jenkins-hbase3.apache.org,45559,1694032716172)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2023-09-06T20:39:08,754 INFO [rs(jenkins-hbase3.apache.org,45559,1694032716172)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1694032748713 on jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:08,754 DEBUG [rs(jenkins-hbase3.apache.org,45559,1694032716172)-backup-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:39:08,755 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50846, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-09-06T20:39:08,774 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-09-06T20:39:08,774 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,45559,1694032716172' in zk 2023-09-06T20:39:08,777 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-09-06T20:39:08,777 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:08,777 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:39:08,777 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:39:08,777 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:39:08,778 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-09-06T20:39:08,779 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:39:08,780 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:39:08,780 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:39:08,781 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:08,781 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:39:08,782 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:39:08,782 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:08,783 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,45559,1694032716172': 2023-09-06T20:39:08,783 INFO [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-09-06T20:39:08,783 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,45559,1694032716172' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-09-06T20:39:08,783 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-09-06T20:39:08,783 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-09-06T20:39:08,783 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-09-06T20:39:08,783 INFO [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-09-06T20:39:08,786 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-09-06T20:39:08,786 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-09-06T20:39:08,786 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:39:08,786 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:39:08,786 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:39:08,787 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:39:08,787 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:39:08,787 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:39:08,787 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:39:08,787 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:39:08,791 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:39:08,793 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:39:08,794 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:08,795 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:39:08,795 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:39:08,796 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:08,809 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:39:08,809 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:39:08,809 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:39:08,809 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-09-06T20:39:08,810 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-09-06T20:39:08,810 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:39:08,810 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:39:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-09-06T20:39:08,920 ERROR [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(228): Unexpected BackupException : Failed stage stage_2 in testing java.io.IOException: Failed stage stage_2 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:211) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.runBackupAndFailAtStage(TestFullBackupWithFailures.java:67) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.testFullBackupWithFailures(TestFullBackupWithFailures.java:55) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:39:08,921 ERROR [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(240): BackupId=backup_1694032747504,startts=1694032748647,failedts=1694032748921,failedphase=REQUEST,failedmessage=Failed stage stage_2 in testing 2023-09-06T20:39:08,921 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-09-06T20:39:08,922 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-09-06T20:39:08,928 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable backup:system 2023-09-06T20:39:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-09-06T20:39:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-09-06T20:39:08,934 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032748934"}]},"ts":"1694032748934"} 2023-09-06T20:39:08,935 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-09-06T20:39:08,937 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-09-06T20:39:08,938 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, UNASSIGN}] 2023-09-06T20:39:08,940 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, UNASSIGN 2023-09-06T20:39:08,941 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=de261a559c0224b9046b4ead35ab8e78, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:08,942 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE; CloseRegionProcedure de261a559c0224b9046b4ead35ab8e78, server=jenkins-hbase3.apache.org,45559,1694032716172}] 2023-09-06T20:39:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-09-06T20:39:09,095 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] handler.UnassignRegionHandler(119): Close de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:09,095 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1668): Closing de261a559c0224b9046b4ead35ab8e78, disabling compactions & flushes 2023-09-06T20:39:09,095 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1690): Closing region backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:09,095 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:09,095 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. after waiting 0 ms 2023-09-06T20:39:09,095 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:09,095 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(2819): Flushing de261a559c0224b9046b4ead35ab8e78 2/2 column families, dataSize=401 B heapSize=1.11 KB 2023-09-06T20:39:09,113 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/meta/2a47b3b149ed4d1d95d3497c6e1e7479 is 116, key is rslogts:hdfs://localhost:34451/backupUT\x00jenkins-hbase3.apache.org:45559/meta:rs-log-ts/1694032748773/Put/seqid=0 2023-09-06T20:39:09,118 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=187 B at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/meta/2a47b3b149ed4d1d95d3497c6e1e7479 2023-09-06T20:39:09,128 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/40b9e689d18b4643828f0a8ecd80edb9 is 218, key is session:backup_1694032747504/session:context/1694032748648/Put/seqid=0 2023-09-06T20:39:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-09-06T20:39:09,533 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=214 B at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/40b9e689d18b4643828f0a8ecd80edb9 2023-09-06T20:39:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-09-06T20:39:09,542 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/meta/2a47b3b149ed4d1d95d3497c6e1e7479 as hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/meta/2a47b3b149ed4d1d95d3497c6e1e7479 2023-09-06T20:39:09,550 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/meta/2a47b3b149ed4d1d95d3497c6e1e7479, entries=2, sequenceid=38, filesize=5.3 K 2023-09-06T20:39:09,551 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/40b9e689d18b4643828f0a8ecd80edb9 as hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/40b9e689d18b4643828f0a8ecd80edb9 2023-09-06T20:39:09,558 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/40b9e689d18b4643828f0a8ecd80edb9, entries=1, sequenceid=38, filesize=5.2 K 2023-09-06T20:39:09,559 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(3022): Finished flush of dataSize ~401 B/401, heapSize ~1.08 KB/1104, currentSize=0 B/0 for de261a559c0224b9046b4ead35ab8e78 in 464ms, sequenceid=38, compaction requested=true 2023-09-06T20:39:09,575 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/recovered.edits/41.seqid, newMaxSeqId=41, maxSeqId=26 2023-09-06T20:39:09,575 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:39:09,576 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1909): Closed backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:09,576 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1622): Region close journal for de261a559c0224b9046b4ead35ab8e78: 2023-09-06T20:39:09,578 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] handler.UnassignRegionHandler(163): Closed de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:09,578 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=de261a559c0224b9046b4ead35ab8e78, regionState=CLOSED 2023-09-06T20:39:09,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=56, resume processing ppid=55 2023-09-06T20:39:09,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=56, ppid=55, state=SUCCESS; CloseRegionProcedure de261a559c0224b9046b4ead35ab8e78, server=jenkins-hbase3.apache.org,45559,1694032716172 in 638 msec 2023-09-06T20:39:09,584 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=55, resume processing ppid=54 2023-09-06T20:39:09,584 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=55, ppid=54, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, UNASSIGN in 644 msec 2023-09-06T20:39:09,585 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032749585"}]},"ts":"1694032749585"} 2023-09-06T20:39:09,586 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-09-06T20:39:09,588 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-09-06T20:39:09,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=54, state=SUCCESS; DisableTableProcedure table=backup:system in 661 msec 2023-09-06T20:39:10,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-09-06T20:39:10,049 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1694032750048 table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:39:10,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032750049 (current time:1694032750049). 2023-09-06T20:39:10,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:39:10,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1694032750048 VERSION not specified, setting to 2 2023-09-06T20:39:10,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d95d48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:39:10,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:39:10,059 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:56746, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:39:10,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:39:10,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:39:10,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=57, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032750048 table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:39:10,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032750048 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 57 2023-09-06T20:39:10,064 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032750048 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:39:10,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=57 2023-09-06T20:39:10,064 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032750048 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:39:10,066 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032750048 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:39:10,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=57 2023-09-06T20:39:10,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=57 2023-09-06T20:39:10,476 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032750048 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-09-06T20:39:10,477 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-09-06T20:39:10,477 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-09-06T20:39:10,477 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:39:10,478 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/meta/2a47b3b149ed4d1d95d3497c6e1e7479] hfiles 2023-09-06T20:39:10,478 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/1): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/meta/2a47b3b149ed4d1d95d3497c6e1e7479 2023-09-06T20:39:10,478 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:39:10,480 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/1e63938d58d1473a8b0b4e3f75f5e1eb, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/40b9e689d18b4643828f0a8ecd80edb9, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8382c9eb13b242258a13aab290003d1c, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8e94389278694c96b296c117230449dd] hfiles 2023-09-06T20:39:10,480 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/4): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/1e63938d58d1473a8b0b4e3f75f5e1eb 2023-09-06T20:39:10,480 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/4): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/40b9e689d18b4643828f0a8ecd80edb9 2023-09-06T20:39:10,480 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/4): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8382c9eb13b242258a13aab290003d1c 2023-09-06T20:39:10,480 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (4/4): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8e94389278694c96b296c117230449dd 2023-09-06T20:39:10,482 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:39:10,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=57 2023-09-06T20:39:10,925 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => de261a559c0224b9046b4ead35ab8e78, NAME => 'backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-09-06T20:39:10,926 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032750048 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:39:10,926 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032750048 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:39:10,927 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1694032750048 2023-09-06T20:39:10,927 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1694032750048 2023-09-06T20:39:11,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=57 2023-09-06T20:39:11,338 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032750048 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:39:11,346 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032750048 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:39:11,347 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1694032750048 to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1694032750048 2023-09-06T20:39:11,348 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032750048 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:39:11,348 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032750048 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 57 2023-09-06T20:39:11,350 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=57, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032750048 table=backup:system type=FLUSH ttl=0 } in 1.2870 sec 2023-09-06T20:39:12,087 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-09-06T20:39:12,091 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52214, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-09-06T20:39:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=57 2023-09-06T20:39:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=58, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032747525 type: FLUSH version: 2 ttl: 0 ) 2023-09-06T20:39:12,183 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(1015): Restore snapshot=snapshot_backup_system as table=backup:system 2023-09-06T20:39:12,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=58 2023-09-06T20:39:12,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=58 2023-09-06T20:39:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=58 2023-09-06T20:39:12,597 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/.tabledesc/.tableinfo.0000000003.635 2023-09-06T20:39:12,597 INFO [PEWorker-4 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/.tabledesc/.tableinfo.0000000004.635 2023-09-06T20:39:12,598 INFO [PEWorker-4 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:39:12,601 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032747525 type: FLUSH version: 2 ttl: 0 2023-09-06T20:39:12,601 DEBUG [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system 2023-09-06T20:39:12,604 DEBUG [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-09-06T20:39:12,604 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(227): region to restore: de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:12,615 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/meta/2a47b3b149ed4d1d95d3497c6e1e7479 to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/archive/data/backup/system/de261a559c0224b9046b4ead35ab8e78/meta/2a47b3b149ed4d1d95d3497c6e1e7479 2023-09-06T20:39:12,615 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:39:12,622 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/40b9e689d18b4643828f0a8ecd80edb9 to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/archive/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/40b9e689d18b4643828f0a8ecd80edb9 2023-09-06T20:39:12,622 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:39:12,622 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032747525 type: FLUSH version: 2 ttl: 0 2023-09-06T20:39:12,624 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1694032752623"}]},"ts":"9223372036854775807"} 2023-09-06T20:39:12,626 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-06T20:39:12,626 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => de261a559c0224b9046b4ead35ab8e78, NAME => 'backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:39:12,626 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1694032752624"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032752624"}]},"ts":"1694032752624"} 2023-09-06T20:39:12,628 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:39:12,628 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-09-06T20:39:12,628 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => de261a559c0224b9046b4ead35ab8e78, NAME => 'backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:39:12,628 INFO [PEWorker-4 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-09-06T20:39:12,630 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=58, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032747525 type: FLUSH version: 2 ttl: 0 ) in 447 msec 2023-09-06T20:39:12,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=58 2023-09-06T20:39:12,790 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1694032750048" 2023-09-06T20:39:12,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(378): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1694032750048 2023-09-06T20:39:12,795 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$11(2631): Client=jenkins//172.31.12.81 enable backup:system 2023-09-06T20:39:12,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=59, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-09-06T20:39:12,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=59 2023-09-06T20:39:12,799 INFO [PEWorker-5 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-09-06T20:39:12,800 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032752800"}]},"ts":"1694032752800"} 2023-09-06T20:39:12,801 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-09-06T20:39:12,805 DEBUG [PEWorker-5 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-09-06T20:39:12,805 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, ASSIGN}] 2023-09-06T20:39:12,807 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, ASSIGN 2023-09-06T20:39:12,808 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-09-06T20:39:12,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=59 2023-09-06T20:39:12,959 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=60 updating hbase:meta row=de261a559c0224b9046b4ead35ab8e78, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:12,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE; OpenRegionProcedure de261a559c0224b9046b4ead35ab8e78, server=jenkins-hbase3.apache.org,45559,1694032716172}] 2023-09-06T20:39:13,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=59 2023-09-06T20:39:13,117 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(130): Open backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:13,117 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7254): Opening region: {ENCODED => de261a559c0224b9046b4ead35ab8e78, NAME => 'backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:39:13,117 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:39:13,118 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:13,118 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(888): Instantiated backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:39:13,118 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7296): checking encryption for de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:13,118 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7299): checking classloading for de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:13,119 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:13,120 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de261a559c0224b9046b4ead35ab8e78 columnFamilyName meta 2023-09-06T20:39:13,120 DEBUG [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:39:13,121 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.HStore(324): Store=de261a559c0224b9046b4ead35ab8e78/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:39:13,121 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:13,122 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de261a559c0224b9046b4ead35ab8e78 columnFamilyName session 2023-09-06T20:39:13,122 DEBUG [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:39:13,131 DEBUG [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/1e63938d58d1473a8b0b4e3f75f5e1eb 2023-09-06T20:39:13,135 DEBUG [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8382c9eb13b242258a13aab290003d1c 2023-09-06T20:39:13,141 DEBUG [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8e94389278694c96b296c117230449dd 2023-09-06T20:39:13,141 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.HStore(324): Store=de261a559c0224b9046b4ead35ab8e78/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:39:13,142 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:13,143 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:13,144 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-09-06T20:39:13,146 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1076): writing seq id for de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:13,147 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): Opened de261a559c0224b9046b4ead35ab8e78; next sequenceid=42; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=12034115040, jitterRate=0.12076430022716522}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-09-06T20:39:13,147 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(992): Region open journal for de261a559c0224b9046b4ead35ab8e78: 2023-09-06T20:39:13,148 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78., pid=61, masterSystemTime=1694032753113 2023-09-06T20:39:13,150 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.CompactSplit(403): Add compact mark for store de261a559c0224b9046b4ead35ab8e78:session, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:39:13,152 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:39:13,153 DEBUG [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:39:13,154 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:13,154 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(158): Opened backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:13,154 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=60 updating hbase:meta row=de261a559c0224b9046b4ead35ab8e78, regionState=OPEN, openSeqNum=42, regionLocation=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:13,156 DEBUG [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 15714 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:39:13,158 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=61, resume processing ppid=60 2023-09-06T20:39:13,158 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=61, ppid=60, state=SUCCESS; OpenRegionProcedure de261a559c0224b9046b4ead35ab8e78, server=jenkins-hbase3.apache.org,45559,1694032716172 in 195 msec 2023-09-06T20:39:13,159 DEBUG [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] regionserver.HStore(1534): de261a559c0224b9046b4ead35ab8e78/session is initiating minor compaction (all files) 2023-09-06T20:39:13,159 INFO [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of de261a559c0224b9046b4ead35ab8e78/session in backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:13,159 INFO [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8e94389278694c96b296c117230449dd, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/1e63938d58d1473a8b0b4e3f75f5e1eb, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8382c9eb13b242258a13aab290003d1c] into tmpdir=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp, totalSize=15.3 K 2023-09-06T20:39:13,160 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=60, resume processing ppid=59 2023-09-06T20:39:13,160 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=60, ppid=59, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, ASSIGN in 353 msec 2023-09-06T20:39:13,161 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032753160"}]},"ts":"1694032753160"} 2023-09-06T20:39:13,161 DEBUG [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] compactions.Compactor(223): Compacting 8e94389278694c96b296c117230449dd, keycount=1, bloomtype=ROW, size=4.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1694032736972 2023-09-06T20:39:13,161 DEBUG [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] compactions.Compactor(223): Compacting 1e63938d58d1473a8b0b4e3f75f5e1eb, keycount=2, bloomtype=ROW, size=5.2 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1694032742622 2023-09-06T20:39:13,162 DEBUG [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] compactions.Compactor(223): Compacting 8382c9eb13b242258a13aab290003d1c, keycount=2, bloomtype=ROW, size=5.2 K, encoding=NONE, compression=NONE, seqNum=32, earliestPutTs=1694032747403 2023-09-06T20:39:13,162 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-09-06T20:39:13,166 INFO [PEWorker-3 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-09-06T20:39:13,170 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=59, state=SUCCESS; EnableTableProcedure table=backup:system in 371 msec 2023-09-06T20:39:13,194 INFO [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de261a559c0224b9046b4ead35ab8e78#session#compaction#8 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:39:13,195 DEBUG [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/21801e1055db41adac61df285d558318 is 256, key is session:backup_1694032736907/session:context/1694032742622/Put/seqid=0 2023-09-06T20:39:13,208 DEBUG [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/21801e1055db41adac61df285d558318 as hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/21801e1055db41adac61df285d558318 2023-09-06T20:39:13,226 INFO [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in de261a559c0224b9046b4ead35ab8e78/session of de261a559c0224b9046b4ead35ab8e78 into 21801e1055db41adac61df285d558318(size=5.6 K), total size for store is 5.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:39:13,226 DEBUG [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for de261a559c0224b9046b4ead35ab8e78: 2023-09-06T20:39:13,227 INFO [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78., storeName=de261a559c0224b9046b4ead35ab8e78/session, priority=13, startTime=1694032753150; duration=0sec 2023-09-06T20:39:13,227 DEBUG [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:39:13,227 DEBUG [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de261a559c0224b9046b4ead35ab8e78:session 2023-09-06T20:39:13,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=59 2023-09-06T20:39:13,404 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(1525): Done restoring backup system table 2023-09-06T20:39:13,404 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:39:13,412 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:39:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:39:13,416 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:39:13,416 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-09-06T20:39:13,421 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:39:13,427 ERROR [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(246): Backup backup_1694032747504 failed. Backup session finished. Status: FAILURE 2023-09-06T20:39:13,427 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:39:13,427 ERROR [Listener at jenkins-hbase3.apache.org/41977 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_2 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:255) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.runBackupAndFailAtStage(TestFullBackupWithFailures.java:67) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.testFullBackupWithFailures(TestFullBackupWithFailures.java:55) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_2 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:211) ~[test-classes/:?] ... 32 more 2023-09-06T20:39:13,471 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] backup.TestFullBackupWithFailures(54): Running stage 3 2023-09-06T20:39:13,495 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a63accf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:39:13,496 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:39:13,498 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52218, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:39:13,505 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:39:13,507 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:40772, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:39:13,568 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:39:13,573 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:39:13,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032753573 (current time:1694032753573). 2023-09-06T20:39:13,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:39:13,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:39:13,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9c925d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:39:13,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:39:13,589 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:40780, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:39:13,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:39:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:39:13,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=62, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:39:13,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-09-06T20:39:13,594 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:39:13,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-09-06T20:39:13,595 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:39:13,597 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:39:13,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-09-06T20:39:13,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-09-06T20:39:14,009 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:39:14,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure de261a559c0224b9046b4ead35ab8e78}] 2023-09-06T20:39:14,011 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:14,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45559 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=63 2023-09-06T20:39:14,163 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:14,164 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegion(2819): Flushing de261a559c0224b9046b4ead35ab8e78 2/2 column families, dataSize=341 B heapSize=1.05 KB 2023-09-06T20:39:14,169 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/9585b49cb83643a185229f157af65936 is 256, key is session:backup_1694032747504/session:context/1694032753417/Put/seqid=0 2023-09-06T20:39:14,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-09-06T20:39:14,575 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=341 B at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/9585b49cb83643a185229f157af65936 2023-09-06T20:39:14,583 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/9585b49cb83643a185229f157af65936 as hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/9585b49cb83643a185229f157af65936 2023-09-06T20:39:14,591 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/9585b49cb83643a185229f157af65936, entries=2, sequenceid=48, filesize=5.2 K 2023-09-06T20:39:14,593 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegion(3022): Finished flush of dataSize ~341 B/341, heapSize ~808 B/808, currentSize=0 B/0 for de261a559c0224b9046b4ead35ab8e78 in 430ms, sequenceid=48, compaction requested=false 2023-09-06T20:39:14,593 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegion(2520): Flush status journal for de261a559c0224b9046b4ead35ab8e78: 2023-09-06T20:39:14,593 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. for snapshot_backup_system completed. 2023-09-06T20:39:14,593 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:39:14,593 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:39:14,593 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-09-06T20:39:14,593 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/21801e1055db41adac61df285d558318, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/9585b49cb83643a185229f157af65936] hfiles 2023-09-06T20:39:14,593 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/21801e1055db41adac61df285d558318 for snapshot=snapshot_backup_system 2023-09-06T20:39:14,594 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/9585b49cb83643a185229f157af65936 for snapshot=snapshot_backup_system 2023-09-06T20:39:14,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-09-06T20:39:15,008 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:15,008 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=63 2023-09-06T20:39:15,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(3995): Remote procedure done, pid=63 2023-09-06T20:39:15,008 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:15,010 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:15,012 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=63, resume processing ppid=62 2023-09-06T20:39:15,012 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:39:15,012 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=63, ppid=62, state=SUCCESS; SnapshotRegionProcedure de261a559c0224b9046b4ead35ab8e78 in 1.0020 sec 2023-09-06T20:39:15,013 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:39:15,014 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:39:15,014 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:39:15,015 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:39:15,036 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:39:15,043 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:39:15,044 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:39:15,045 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:39:15,045 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-09-06T20:39:15,046 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=62, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.4540 sec 2023-09-06T20:39:15,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-09-06T20:39:15,703 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(115): Backup backup_1694032753545 started at 1694032755703. 2023-09-06T20:39:15,705 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(119): Backup session backup_1694032753545 has been started. 2023-09-06T20:39:15,707 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] backup.TestBackupBase$FullTableBackupClientForTest(205): Execute roll log procedure for full backup ... 2023-09-06T20:39:15,707 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-09-06T20:39:15,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-09-06T20:39:15,708 INFO [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-09-06T20:39:15,708 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-09-06T20:39:15,709 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-09-06T20:39:15,709 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-09-06T20:39:15,710 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:39:15,711 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:15,711 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:39:15,711 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:39:15,711 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-09-06T20:39:15,712 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-09-06T20:39:15,712 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-09-06T20:39:15,712 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:39:15,712 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-09-06T20:39:15,713 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-09-06T20:39:15,713 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-09-06T20:39:15,713 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-09-06T20:39:15,713 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-09-06T20:39:15,714 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-09-06T20:39:15,714 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-09-06T20:39:15,714 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,45559,1694032716172' joining acquired barrier for procedure (rolllog) in zk 2023-09-06T20:39:15,717 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-09-06T20:39:15,717 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:15,717 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:39:15,717 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:39:15,717 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-09-06T20:39:15,717 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:39:15,717 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:39:15,718 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:39:15,718 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:15,719 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:39:15,719 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,45559,1694032716172' joining acquired barrier for procedure 'rolllog' on coordinator 2023-09-06T20:39:15,719 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@7762c9ec[Count = 0] remaining members to acquire global barrier 2023-09-06T20:39:15,719 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-09-06T20:39:15,719 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-09-06T20:39:15,720 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-09-06T20:39:15,720 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-09-06T20:39:15,720 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-09-06T20:39:15,721 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-09-06T20:39:15,722 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-09-06T20:39:15,722 DEBUG [rs(jenkins-hbase3.apache.org,45559,1694032716172)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:15,722 INFO [rs(jenkins-hbase3.apache.org,45559,1694032716172)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1694032748713 highest: 1694032748713 on jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:15,722 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45559%2C1694032716172.meta:.meta(num 1694032748693) roll requested 2023-09-06T20:39:15,733 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.meta.1694032755722.meta, exclude list is [], retry=0 2023-09-06T20:39:15,738 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41237,DS-07aa54ab-1cb2-4831-8bde-a715b49e632a,DISK] 2023-09-06T20:39:15,742 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.meta.1694032748693.meta with entries=10, filesize=2.50 KB; new WAL /user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.meta.1694032755722.meta 2023-09-06T20:39:15,742 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41237,DS-07aa54ab-1cb2-4831-8bde-a715b49e632a,DISK]] 2023-09-06T20:39:15,742 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.meta.1694032748693.meta is not closed yet, will try archiving it next time 2023-09-06T20:39:15,742 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45559%2C1694032716172:(num 1694032748713) roll requested 2023-09-06T20:39:15,744 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.meta.1694032748693.meta not finished, retry = 0 2023-09-06T20:39:15,754 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.1694032755742, exclude list is [], retry=0 2023-09-06T20:39:15,759 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41237,DS-07aa54ab-1cb2-4831-8bde-a715b49e632a,DISK] 2023-09-06T20:39:15,760 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.1694032748713 with entries=13, filesize=3.39 KB; new WAL /user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.1694032755742 2023-09-06T20:39:15,761 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41237,DS-07aa54ab-1cb2-4831-8bde-a715b49e632a,DISK]] 2023-09-06T20:39:15,761 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.1694032748713 is not closed yet, will try archiving it next time 2023-09-06T20:39:15,762 DEBUG [rs(jenkins-hbase3.apache.org,45559,1694032716172)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-09-06T20:39:15,762 INFO [rs(jenkins-hbase3.apache.org,45559,1694032716172)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1694032755742 on jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:15,763 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.1694032748713 not finished, retry = 0 2023-09-06T20:39:15,776 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-09-06T20:39:15,776 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,45559,1694032716172' in zk 2023-09-06T20:39:15,780 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-09-06T20:39:15,780 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:15,780 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:39:15,780 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:39:15,780 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:39:15,780 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-09-06T20:39:15,781 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:39:15,782 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:39:15,782 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:39:15,782 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:15,783 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:39:15,783 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:39:15,783 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:15,785 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,45559,1694032716172': 2023-09-06T20:39:15,785 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,45559,1694032716172' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-09-06T20:39:15,785 INFO [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-09-06T20:39:15,785 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-09-06T20:39:15,785 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-09-06T20:39:15,785 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-09-06T20:39:15,785 INFO [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-09-06T20:39:15,787 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-09-06T20:39:15,787 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:39:15,788 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:39:15,788 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-09-06T20:39:15,788 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:39:15,788 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:39:15,788 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:39:15,788 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:39:15,789 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:39:15,789 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:39:15,789 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:39:15,790 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:39:15,790 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:15,791 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:39:15,791 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:39:15,792 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:15,795 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:39:15,796 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:39:15,796 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:39:15,796 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-09-06T20:39:15,797 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-09-06T20:39:15,797 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:39:15,797 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:39:15,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-09-06T20:39:15,901 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1694032755900_default_test-1694032725153 table=test-1694032725153 type=FLUSH ttl=0 } 2023-09-06T20:39:15,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032755901 (current time:1694032755901). 2023-09-06T20:39:15,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:39:15,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1694032755900_default_test-1694032725153 VERSION not specified, setting to 2 2023-09-06T20:39:15,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@df4ee68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:39:15,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:39:15,910 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:40794, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:39:15,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:39:15,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:39:15,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=64, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1694032755900_default_test-1694032725153 table=test-1694032725153 type=FLUSH ttl=0 } 2023-09-06T20:39:15,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1694032755900_default_test-1694032725153 table=test-1694032725153 type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-09-06T20:39:15,914 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1694032755900_default_test-1694032725153 table=test-1694032725153 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:39:15,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-09-06T20:39:15,915 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1694032755900_default_test-1694032725153 table=test-1694032725153 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:39:15,917 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1694032755900_default_test-1694032725153 table=test-1694032725153 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:39:16,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-09-06T20:39:16,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-09-06T20:39:16,324 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1694032755900_default_test-1694032725153 table=test-1694032725153 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:39:16,324 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 7499612530301bea74e2d6f06f601777}] 2023-09-06T20:39:16,326 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 7499612530301bea74e2d6f06f601777 2023-09-06T20:39:16,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45559 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=65 2023-09-06T20:39:16,478 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1694032725153,,1694032725664.7499612530301bea74e2d6f06f601777. 2023-09-06T20:39:16,480 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2819): Flushing 7499612530301bea74e2d6f06f601777 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-09-06T20:39:16,496 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/default/test-1694032725153/7499612530301bea74e2d6f06f601777/.tmp/f/fb7f27fc980f49db93fa2308b21c456f is 37, key is row10/f:q1/1694032727828/Put/seqid=0 2023-09-06T20:39:16,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-09-06T20:39:16,901 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/default/test-1694032725153/7499612530301bea74e2d6f06f601777/.tmp/f/fb7f27fc980f49db93fa2308b21c456f 2023-09-06T20:39:16,908 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/default/test-1694032725153/7499612530301bea74e2d6f06f601777/.tmp/f/fb7f27fc980f49db93fa2308b21c456f as hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/default/test-1694032725153/7499612530301bea74e2d6f06f601777/f/fb7f27fc980f49db93fa2308b21c456f 2023-09-06T20:39:16,913 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/default/test-1694032725153/7499612530301bea74e2d6f06f601777/f/fb7f27fc980f49db93fa2308b21c456f, entries=99, sequenceid=103, filesize=8.2 K 2023-09-06T20:39:16,914 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 7499612530301bea74e2d6f06f601777 in 434ms, sequenceid=103, compaction requested=false 2023-09-06T20:39:16,915 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2520): Flush status journal for 7499612530301bea74e2d6f06f601777: 2023-09-06T20:39:16,915 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1694032725153,,1694032725664.7499612530301bea74e2d6f06f601777. for snapshot_1694032755900_default_test-1694032725153 completed. 2023-09-06T20:39:16,915 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(240): Storing 'test-1694032725153,,1694032725664.7499612530301bea74e2d6f06f601777.' region-info for snapshot=snapshot_1694032755900_default_test-1694032725153 2023-09-06T20:39:16,915 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:39:16,915 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/default/test-1694032725153/7499612530301bea74e2d6f06f601777/f/fb7f27fc980f49db93fa2308b21c456f] hfiles 2023-09-06T20:39:16,915 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/default/test-1694032725153/7499612530301bea74e2d6f06f601777/f/fb7f27fc980f49db93fa2308b21c456f for snapshot=snapshot_1694032755900_default_test-1694032725153 2023-09-06T20:39:17,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-09-06T20:39:17,321 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1694032725153,,1694032725664.7499612530301bea74e2d6f06f601777. 2023-09-06T20:39:17,321 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=65 2023-09-06T20:39:17,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(3995): Remote procedure done, pid=65 2023-09-06T20:39:17,321 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1694032755900_default_test-1694032725153 on region 7499612530301bea74e2d6f06f601777 2023-09-06T20:39:17,323 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 7499612530301bea74e2d6f06f601777 2023-09-06T20:39:17,325 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=65, resume processing ppid=64 2023-09-06T20:39:17,325 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=65, ppid=64, state=SUCCESS; SnapshotRegionProcedure 7499612530301bea74e2d6f06f601777 in 999 msec 2023-09-06T20:39:17,325 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1694032755900_default_test-1694032725153 table=test-1694032725153 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:39:17,326 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1694032755900_default_test-1694032725153 table=test-1694032725153 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:39:17,326 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1694032755900_default_test-1694032725153 table=test-1694032725153 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:39:17,326 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1694032755900_default_test-1694032725153 2023-09-06T20:39:17,327 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/.tmp/snapshot_1694032755900_default_test-1694032725153 2023-09-06T20:39:17,739 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1694032755900_default_test-1694032725153 table=test-1694032725153 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:39:17,745 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1694032755900_default_test-1694032725153 table=test-1694032725153 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:39:17,745 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/.tmp/snapshot_1694032755900_default_test-1694032725153 to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/snapshot_1694032755900_default_test-1694032725153 2023-09-06T20:39:17,746 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1694032755900_default_test-1694032725153 table=test-1694032725153 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:39:17,746 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1694032755900_default_test-1694032725153 table=test-1694032725153 type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-09-06T20:39:17,747 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=64, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1694032755900_default_test-1694032725153 table=test-1694032725153 type=FLUSH ttl=0 } in 1.8340 sec 2023-09-06T20:39:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-09-06T20:39:18,022 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1694032758022_ns2_test-16940327251531 table=ns2:test-16940327251531 type=FLUSH ttl=0 } 2023-09-06T20:39:18,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032758022 (current time:1694032758022). 2023-09-06T20:39:18,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:39:18,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1694032758022_ns2_test-16940327251531 VERSION not specified, setting to 2 2023-09-06T20:39:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27700c33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:39:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:39:18,033 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:40800, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:39:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:39:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:39:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=66, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1694032758022_ns2_test-16940327251531 table=ns2:test-16940327251531 type=FLUSH ttl=0 } 2023-09-06T20:39:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1694032758022_ns2_test-16940327251531 table=ns2:test-16940327251531 type=FLUSH ttl=0 }, snapshot procedure id = 66 2023-09-06T20:39:18,036 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1694032758022_ns2_test-16940327251531 table=ns2:test-16940327251531 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:39:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-09-06T20:39:18,037 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1694032758022_ns2_test-16940327251531 table=ns2:test-16940327251531 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:39:18,039 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1694032758022_ns2_test-16940327251531 table=ns2:test-16940327251531 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:39:18,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-09-06T20:39:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-09-06T20:39:18,445 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1694032758022_ns2_test-16940327251531 table=ns2:test-16940327251531 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:39:18,445 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure 87e84cabad1c9957eead562b206d922d}] 2023-09-06T20:39:18,448 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure 87e84cabad1c9957eead562b206d922d 2023-09-06T20:39:18,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45559 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=67 2023-09-06T20:39:18,600 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16940327251531,,1694032727923.87e84cabad1c9957eead562b206d922d. 2023-09-06T20:39:18,602 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HRegion(2819): Flushing 87e84cabad1c9957eead562b206d922d 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-09-06T20:39:18,616 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/ns2/test-16940327251531/87e84cabad1c9957eead562b206d922d/.tmp/f/e976e7f8d3c44b73991e878da4fa6496 is 37, key is row10/f:q1/1694032729051/Put/seqid=0 2023-09-06T20:39:18,621 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:39:18,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-09-06T20:39:19,021 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/ns2/test-16940327251531/87e84cabad1c9957eead562b206d922d/.tmp/f/e976e7f8d3c44b73991e878da4fa6496 2023-09-06T20:39:19,028 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/ns2/test-16940327251531/87e84cabad1c9957eead562b206d922d/.tmp/f/e976e7f8d3c44b73991e878da4fa6496 as hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/ns2/test-16940327251531/87e84cabad1c9957eead562b206d922d/f/e976e7f8d3c44b73991e878da4fa6496 2023-09-06T20:39:19,034 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/ns2/test-16940327251531/87e84cabad1c9957eead562b206d922d/f/e976e7f8d3c44b73991e878da4fa6496, entries=99, sequenceid=103, filesize=8.2 K 2023-09-06T20:39:19,035 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 87e84cabad1c9957eead562b206d922d in 433ms, sequenceid=103, compaction requested=false 2023-09-06T20:39:19,036 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HRegion(2520): Flush status journal for 87e84cabad1c9957eead562b206d922d: 2023-09-06T20:39:19,036 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16940327251531,,1694032727923.87e84cabad1c9957eead562b206d922d. for snapshot_1694032758022_ns2_test-16940327251531 completed. 2023-09-06T20:39:19,036 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16940327251531,,1694032727923.87e84cabad1c9957eead562b206d922d.' region-info for snapshot=snapshot_1694032758022_ns2_test-16940327251531 2023-09-06T20:39:19,036 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:39:19,036 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/ns2/test-16940327251531/87e84cabad1c9957eead562b206d922d/f/e976e7f8d3c44b73991e878da4fa6496] hfiles 2023-09-06T20:39:19,036 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/ns2/test-16940327251531/87e84cabad1c9957eead562b206d922d/f/e976e7f8d3c44b73991e878da4fa6496 for snapshot=snapshot_1694032758022_ns2_test-16940327251531 2023-09-06T20:39:19,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-09-06T20:39:19,442 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16940327251531,,1694032727923.87e84cabad1c9957eead562b206d922d. 2023-09-06T20:39:19,442 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=67 2023-09-06T20:39:19,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(3995): Remote procedure done, pid=67 2023-09-06T20:39:19,442 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1694032758022_ns2_test-16940327251531 on region 87e84cabad1c9957eead562b206d922d 2023-09-06T20:39:19,444 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure 87e84cabad1c9957eead562b206d922d 2023-09-06T20:39:19,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=67, resume processing ppid=66 2023-09-06T20:39:19,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=67, ppid=66, state=SUCCESS; SnapshotRegionProcedure 87e84cabad1c9957eead562b206d922d in 1.0000 sec 2023-09-06T20:39:19,446 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1694032758022_ns2_test-16940327251531 table=ns2:test-16940327251531 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:39:19,447 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1694032758022_ns2_test-16940327251531 table=ns2:test-16940327251531 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:39:19,448 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1694032758022_ns2_test-16940327251531 table=ns2:test-16940327251531 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:39:19,448 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1694032758022_ns2_test-16940327251531 2023-09-06T20:39:19,448 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/.tmp/snapshot_1694032758022_ns2_test-16940327251531 2023-09-06T20:39:19,859 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1694032758022_ns2_test-16940327251531 table=ns2:test-16940327251531 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:39:19,865 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1694032758022_ns2_test-16940327251531 table=ns2:test-16940327251531 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:39:19,865 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/.tmp/snapshot_1694032758022_ns2_test-16940327251531 to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/snapshot_1694032758022_ns2_test-16940327251531 2023-09-06T20:39:19,866 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1694032758022_ns2_test-16940327251531 table=ns2:test-16940327251531 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:39:19,866 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1694032758022_ns2_test-16940327251531 table=ns2:test-16940327251531 type=FLUSH ttl=0 }, snapshot procedure id = 66 2023-09-06T20:39:19,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=66, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1694032758022_ns2_test-16940327251531 table=ns2:test-16940327251531 type=FLUSH ttl=0 } in 1.8320 sec 2023-09-06T20:39:20,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-09-06T20:39:20,142 ERROR [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(228): Unexpected BackupException : Failed stage stage_3 in testing java.io.IOException: Failed stage stage_3 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:223) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.runBackupAndFailAtStage(TestFullBackupWithFailures.java:67) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.testFullBackupWithFailures(TestFullBackupWithFailures.java:55) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:39:20,143 ERROR [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(240): BackupId=backup_1694032753545,startts=1694032755703,failedts=1694032760143,failedphase=SNAPSHOT,failedmessage=Failed stage stage_3 in testing 2023-09-06T20:39:20,143 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-09-06T20:39:20,143 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1694032758022_ns2_test-16940327251531 2023-09-06T20:39:20,144 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1694032758022_ns2_test-16940327251531" 2023-09-06T20:39:20,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1694032758022_ns2_test-16940327251531 2023-09-06T20:39:20,147 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1694032758022_ns2_test-16940327251531 for backup backup_1694032753545 succeeded. 2023-09-06T20:39:20,147 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1694032755900_default_test-1694032725153 2023-09-06T20:39:20,148 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1694032755900_default_test-1694032725153" 2023-09-06T20:39:20,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1694032755900_default_test-1694032725153 2023-09-06T20:39:20,152 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1694032755900_default_test-1694032725153 for backup backup_1694032753545 succeeded. 2023-09-06T20:39:20,152 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-09-06T20:39:20,157 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable backup:system 2023-09-06T20:39:20,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=68, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-09-06T20:39:20,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-09-06T20:39:20,163 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032760162"}]},"ts":"1694032760162"} 2023-09-06T20:39:20,164 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-09-06T20:39:20,167 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-09-06T20:39:20,168 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, UNASSIGN}] 2023-09-06T20:39:20,170 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, UNASSIGN 2023-09-06T20:39:20,171 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=69 updating hbase:meta row=de261a559c0224b9046b4ead35ab8e78, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:20,172 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE; CloseRegionProcedure de261a559c0224b9046b4ead35ab8e78, server=jenkins-hbase3.apache.org,45559,1694032716172}] 2023-09-06T20:39:20,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-09-06T20:39:20,324 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] handler.UnassignRegionHandler(119): Close de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:20,324 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1668): Closing de261a559c0224b9046b4ead35ab8e78, disabling compactions & flushes 2023-09-06T20:39:20,324 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1690): Closing region backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:20,324 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:20,324 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. after waiting 0 ms 2023-09-06T20:39:20,324 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:20,325 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(2819): Flushing de261a559c0224b9046b4ead35ab8e78 2/2 column families, dataSize=401 B heapSize=1.11 KB 2023-09-06T20:39:20,341 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/meta/a73046953ae94cf1990900d5039ea6f3 is 116, key is rslogts:hdfs://localhost:34451/backupUT\x00jenkins-hbase3.apache.org:45559/meta:rs-log-ts/1694032755775/Put/seqid=0 2023-09-06T20:39:20,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-09-06T20:39:20,746 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=187 B at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/meta/a73046953ae94cf1990900d5039ea6f3 2023-09-06T20:39:20,755 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/32a4c5957e744d368b17d8d68d0ce08b is 218, key is session:backup_1694032753545/session:context/1694032755703/Put/seqid=0 2023-09-06T20:39:20,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-09-06T20:39:21,159 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=214 B at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/32a4c5957e744d368b17d8d68d0ce08b 2023-09-06T20:39:21,166 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/meta/a73046953ae94cf1990900d5039ea6f3 as hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/meta/a73046953ae94cf1990900d5039ea6f3 2023-09-06T20:39:21,172 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/meta/a73046953ae94cf1990900d5039ea6f3, entries=2, sequenceid=54, filesize=5.3 K 2023-09-06T20:39:21,173 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/32a4c5957e744d368b17d8d68d0ce08b as hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/32a4c5957e744d368b17d8d68d0ce08b 2023-09-06T20:39:21,178 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/32a4c5957e744d368b17d8d68d0ce08b, entries=1, sequenceid=54, filesize=5.2 K 2023-09-06T20:39:21,179 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(3022): Finished flush of dataSize ~401 B/401, heapSize ~1.08 KB/1104, currentSize=0 B/0 for de261a559c0224b9046b4ead35ab8e78 in 855ms, sequenceid=54, compaction requested=true 2023-09-06T20:39:21,186 DEBUG [StoreCloser-backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8e94389278694c96b296c117230449dd, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/1e63938d58d1473a8b0b4e3f75f5e1eb, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8382c9eb13b242258a13aab290003d1c] to archive 2023-09-06T20:39:21,186 DEBUG [StoreCloser-backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-06T20:39:21,190 DEBUG [StoreCloser-backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8e94389278694c96b296c117230449dd to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/archive/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8e94389278694c96b296c117230449dd 2023-09-06T20:39:21,192 DEBUG [StoreCloser-backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/1e63938d58d1473a8b0b4e3f75f5e1eb to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/archive/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/1e63938d58d1473a8b0b4e3f75f5e1eb 2023-09-06T20:39:21,193 DEBUG [StoreCloser-backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8382c9eb13b242258a13aab290003d1c to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/archive/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/8382c9eb13b242258a13aab290003d1c 2023-09-06T20:39:21,228 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/recovered.edits/57.seqid, newMaxSeqId=57, maxSeqId=41 2023-09-06T20:39:21,229 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:39:21,229 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1909): Closed backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:21,229 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1622): Region close journal for de261a559c0224b9046b4ead35ab8e78: 2023-09-06T20:39:21,231 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] handler.UnassignRegionHandler(163): Closed de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:21,231 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=69 updating hbase:meta row=de261a559c0224b9046b4ead35ab8e78, regionState=CLOSED 2023-09-06T20:39:21,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=70, resume processing ppid=69 2023-09-06T20:39:21,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=70, ppid=69, state=SUCCESS; CloseRegionProcedure de261a559c0224b9046b4ead35ab8e78, server=jenkins-hbase3.apache.org,45559,1694032716172 in 1.0610 sec 2023-09-06T20:39:21,235 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=69, resume processing ppid=68 2023-09-06T20:39:21,235 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=69, ppid=68, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, UNASSIGN in 1.0660 sec 2023-09-06T20:39:21,236 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032761236"}]},"ts":"1694032761236"} 2023-09-06T20:39:21,237 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-09-06T20:39:21,239 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-09-06T20:39:21,240 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=68, state=SUCCESS; DisableTableProcedure table=backup:system in 1.0820 sec 2023-09-06T20:39:21,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-09-06T20:39:21,275 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1694032761274 table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:39:21,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032761275 (current time:1694032761275). 2023-09-06T20:39:21,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:39:21,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1694032761274 VERSION not specified, setting to 2 2023-09-06T20:39:21,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d92032e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:39:21,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:39:21,288 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:33498, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:39:21,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:39:21,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:39:21,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=71, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032761274 table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:39:21,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032761274 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 71 2023-09-06T20:39:21,292 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032761274 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:39:21,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=71 2023-09-06T20:39:21,292 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032761274 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:39:21,294 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032761274 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:39:21,328 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-09-06T20:39:21,328 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-09-06T20:39:21,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=71 2023-09-06T20:39:21,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=71 2023-09-06T20:39:21,701 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032761274 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-09-06T20:39:21,702 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-09-06T20:39:21,703 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-09-06T20:39:21,703 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:39:21,704 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/meta/a73046953ae94cf1990900d5039ea6f3] hfiles 2023-09-06T20:39:21,704 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/1): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/meta/a73046953ae94cf1990900d5039ea6f3 2023-09-06T20:39:21,704 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:39:21,705 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/21801e1055db41adac61df285d558318, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/32a4c5957e744d368b17d8d68d0ce08b, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/9585b49cb83643a185229f157af65936] hfiles 2023-09-06T20:39:21,705 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/3): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/21801e1055db41adac61df285d558318 2023-09-06T20:39:21,705 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/3): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/32a4c5957e744d368b17d8d68d0ce08b 2023-09-06T20:39:21,705 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/3): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/9585b49cb83643a185229f157af65936 2023-09-06T20:39:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=71 2023-09-06T20:39:22,111 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => de261a559c0224b9046b4ead35ab8e78, NAME => 'backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-09-06T20:39:22,112 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032761274 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:39:22,113 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032761274 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:39:22,113 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1694032761274 2023-09-06T20:39:22,114 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1694032761274 2023-09-06T20:39:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=71 2023-09-06T20:39:22,524 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032761274 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:39:22,532 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032761274 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:39:22,533 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1694032761274 to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1694032761274 2023-09-06T20:39:22,534 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032761274 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:39:22,534 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032761274 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 71 2023-09-06T20:39:22,535 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=71, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032761274 table=backup:system type=FLUSH ttl=0 } in 1.2450 sec 2023-09-06T20:39:23,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=71 2023-09-06T20:39:23,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=72, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032753573 type: FLUSH version: 2 ttl: 0 ) 2023-09-06T20:39:23,405 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(1015): Restore snapshot=snapshot_backup_system as table=backup:system 2023-09-06T20:39:23,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-09-06T20:39:23,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-09-06T20:39:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-09-06T20:39:23,819 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/.tabledesc/.tableinfo.0000000004.635 2023-09-06T20:39:23,819 INFO [PEWorker-3 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/.tabledesc/.tableinfo.0000000005.635 2023-09-06T20:39:23,820 INFO [PEWorker-3 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:39:23,823 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032753573 type: FLUSH version: 2 ttl: 0 2023-09-06T20:39:23,824 DEBUG [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system 2023-09-06T20:39:23,827 DEBUG [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-09-06T20:39:23,827 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(227): region to restore: de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:23,837 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/meta/a73046953ae94cf1990900d5039ea6f3 to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/archive/data/backup/system/de261a559c0224b9046b4ead35ab8e78/meta/a73046953ae94cf1990900d5039ea6f3 2023-09-06T20:39:23,837 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:39:23,840 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/32a4c5957e744d368b17d8d68d0ce08b to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/archive/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/32a4c5957e744d368b17d8d68d0ce08b 2023-09-06T20:39:23,841 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:39:23,841 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032753573 type: FLUSH version: 2 ttl: 0 2023-09-06T20:39:23,842 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1694032763842"}]},"ts":"9223372036854775807"} 2023-09-06T20:39:23,844 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-06T20:39:23,844 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => de261a559c0224b9046b4ead35ab8e78, NAME => 'backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:39:23,844 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1694032763843"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032763843"}]},"ts":"1694032763843"} 2023-09-06T20:39:23,846 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:39:23,846 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-09-06T20:39:23,846 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => de261a559c0224b9046b4ead35ab8e78, NAME => 'backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:39:23,846 INFO [PEWorker-3 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-09-06T20:39:23,848 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=72, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032753573 type: FLUSH version: 2 ttl: 0 ) in 443 msec 2023-09-06T20:39:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-09-06T20:39:24,012 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1694032761274" 2023-09-06T20:39:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(378): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1694032761274 2023-09-06T20:39:24,016 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$11(2631): Client=jenkins//172.31.12.81 enable backup:system 2023-09-06T20:39:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=73, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-09-06T20:39:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=73 2023-09-06T20:39:24,019 INFO [PEWorker-1 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-09-06T20:39:24,019 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032764019"}]},"ts":"1694032764019"} 2023-09-06T20:39:24,021 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-09-06T20:39:24,023 DEBUG [PEWorker-1 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-09-06T20:39:24,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, ASSIGN}] 2023-09-06T20:39:24,025 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, ASSIGN 2023-09-06T20:39:24,025 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-09-06T20:39:24,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=73 2023-09-06T20:39:24,177 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=74 updating hbase:meta row=de261a559c0224b9046b4ead35ab8e78, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:24,179 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE; OpenRegionProcedure de261a559c0224b9046b4ead35ab8e78, server=jenkins-hbase3.apache.org,45559,1694032716172}] 2023-09-06T20:39:24,270 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-09-06T20:39:24,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=73 2023-09-06T20:39:24,333 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(130): Open backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:24,334 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7254): Opening region: {ENCODED => de261a559c0224b9046b4ead35ab8e78, NAME => 'backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:39:24,334 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:39:24,334 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:24,334 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(888): Instantiated backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:39:24,334 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7296): checking encryption for de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:24,334 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7299): checking classloading for de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:24,336 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:24,336 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de261a559c0224b9046b4ead35ab8e78 columnFamilyName meta 2023-09-06T20:39:24,336 DEBUG [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:39:24,337 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.HStore(324): Store=de261a559c0224b9046b4ead35ab8e78/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:39:24,337 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:24,338 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de261a559c0224b9046b4ead35ab8e78 columnFamilyName session 2023-09-06T20:39:24,338 DEBUG [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:39:24,343 DEBUG [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/21801e1055db41adac61df285d558318 2023-09-06T20:39:24,347 DEBUG [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/9585b49cb83643a185229f157af65936 2023-09-06T20:39:24,348 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.HStore(324): Store=de261a559c0224b9046b4ead35ab8e78/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:39:24,348 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:24,349 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:24,351 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-09-06T20:39:24,352 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1076): writing seq id for de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:24,353 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): Opened de261a559c0224b9046b4ead35ab8e78; next sequenceid=58; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9691967840, jitterRate=-0.09736515581607819}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-09-06T20:39:24,353 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(992): Region open journal for de261a559c0224b9046b4ead35ab8e78: 2023-09-06T20:39:24,354 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78., pid=75, masterSystemTime=1694032764330 2023-09-06T20:39:24,355 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:24,355 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(158): Opened backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:24,356 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=74 updating hbase:meta row=de261a559c0224b9046b4ead35ab8e78, regionState=OPEN, openSeqNum=58, regionLocation=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:24,359 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=75, resume processing ppid=74 2023-09-06T20:39:24,359 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=75, ppid=74, state=SUCCESS; OpenRegionProcedure de261a559c0224b9046b4ead35ab8e78, server=jenkins-hbase3.apache.org,45559,1694032716172 in 178 msec 2023-09-06T20:39:24,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=74, resume processing ppid=73 2023-09-06T20:39:24,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=74, ppid=73, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, ASSIGN in 336 msec 2023-09-06T20:39:24,360 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032764360"}]},"ts":"1694032764360"} 2023-09-06T20:39:24,362 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-09-06T20:39:24,363 INFO [PEWorker-2 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-09-06T20:39:24,364 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=73, state=SUCCESS; EnableTableProcedure table=backup:system in 348 msec 2023-09-06T20:39:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=73 2023-09-06T20:39:24,622 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(1525): Done restoring backup system table 2023-09-06T20:39:24,622 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:39:24,627 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:39:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:39:24,629 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:39:24,630 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: SNAPSHOT 2023-09-06T20:39:24,631 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:39:24,636 ERROR [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(246): Backup backup_1694032753545 failed. Backup session finished. Status: FAILURE 2023-09-06T20:39:24,637 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:39:24,637 ERROR [Listener at jenkins-hbase3.apache.org/41977 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_3 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:255) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.runBackupAndFailAtStage(TestFullBackupWithFailures.java:67) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.testFullBackupWithFailures(TestFullBackupWithFailures.java:55) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_3 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:223) ~[test-classes/:?] ... 32 more 2023-09-06T20:39:24,682 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] backup.TestFullBackupWithFailures(54): Running stage 4 2023-09-06T20:39:24,702 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@486f70ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:39:24,703 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-09-06T20:39:24,704 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:43800, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-09-06T20:39:24,707 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:39:24,708 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:33514, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:39:24,742 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-09-06T20:39:24,745 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:39:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032764745 (current time:1694032764745). 2023-09-06T20:39:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:39:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-09-06T20:39:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50b4dda8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:39:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:39:24,755 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:33528, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:39:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:39:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:39:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=76, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=76, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:39:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 76 2023-09-06T20:39:24,759 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=76, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=76, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:39:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=76 2023-09-06T20:39:24,760 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=76, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=76, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:39:24,761 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=76, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=76, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:39:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=76 2023-09-06T20:39:25,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=76 2023-09-06T20:39:25,168 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=76, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=76, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:39:25,168 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE; SnapshotRegionProcedure de261a559c0224b9046b4ead35ab8e78}] 2023-09-06T20:39:25,170 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=77, ppid=76, state=RUNNABLE; SnapshotRegionProcedure de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:25,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45559 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=77 2023-09-06T20:39:25,321 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:25,323 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] regionserver.HRegion(2819): Flushing de261a559c0224b9046b4ead35ab8e78 2/2 column families, dataSize=440 B heapSize=1.15 KB 2023-09-06T20:39:25,336 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/98a5e9c6fde5479191b554626ed8f3c4 is 355, key is session:backup_1694032753545/session:context/1694032764630/Put/seqid=0 2023-09-06T20:39:25,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=76 2023-09-06T20:39:25,741 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=440 B at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/98a5e9c6fde5479191b554626ed8f3c4 2023-09-06T20:39:25,747 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/98a5e9c6fde5479191b554626ed8f3c4 as hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/98a5e9c6fde5479191b554626ed8f3c4 2023-09-06T20:39:25,752 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/98a5e9c6fde5479191b554626ed8f3c4, entries=2, sequenceid=63, filesize=5.3 K 2023-09-06T20:39:25,753 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] regionserver.HRegion(3022): Finished flush of dataSize ~440 B/440, heapSize ~904 B/904, currentSize=0 B/0 for de261a559c0224b9046b4ead35ab8e78 in 430ms, sequenceid=63, compaction requested=true 2023-09-06T20:39:25,753 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] regionserver.HRegion(2520): Flush status journal for de261a559c0224b9046b4ead35ab8e78: 2023-09-06T20:39:25,753 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. for snapshot_backup_system completed. 2023-09-06T20:39:25,753 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.' region-info for snapshot=snapshot_backup_system 2023-09-06T20:39:25,753 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:39:25,753 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-09-06T20:39:25,753 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/21801e1055db41adac61df285d558318, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/9585b49cb83643a185229f157af65936, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/98a5e9c6fde5479191b554626ed8f3c4] hfiles 2023-09-06T20:39:25,753 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/21801e1055db41adac61df285d558318 for snapshot=snapshot_backup_system 2023-09-06T20:39:25,753 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/9585b49cb83643a185229f157af65936 for snapshot=snapshot_backup_system 2023-09-06T20:39:25,753 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/98a5e9c6fde5479191b554626ed8f3c4 for snapshot=snapshot_backup_system 2023-09-06T20:39:25,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=76 2023-09-06T20:39:26,159 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:26,159 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=77 2023-09-06T20:39:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(3995): Remote procedure done, pid=77 2023-09-06T20:39:26,160 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:26,161 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=77, ppid=76, state=RUNNABLE; SnapshotRegionProcedure de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:26,163 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=77, resume processing ppid=76 2023-09-06T20:39:26,163 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=76, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=76, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:39:26,163 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=77, ppid=76, state=SUCCESS; SnapshotRegionProcedure de261a559c0224b9046b4ead35ab8e78 in 994 msec 2023-09-06T20:39:26,164 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=76, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=76, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:39:26,164 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=76, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=76, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:39:26,164 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-09-06T20:39:26,165 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/.tmp/snapshot_backup_system 2023-09-06T20:39:26,581 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=76, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=76, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:39:26,587 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=76, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=76, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:39:26,588 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/snapshot_backup_system 2023-09-06T20:39:26,589 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=76, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=76, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:39:26,589 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 76 2023-09-06T20:39:26,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=76, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=76, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8320 sec 2023-09-06T20:39:26,732 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:39:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=76 2023-09-06T20:39:26,866 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(115): Backup backup_1694032764728 started at 1694032766866. 2023-09-06T20:39:26,868 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(119): Backup session backup_1694032764728 has been started. 2023-09-06T20:39:26,870 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] backup.TestBackupBase$FullTableBackupClientForTest(205): Execute roll log procedure for full backup ... 2023-09-06T20:39:26,870 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-09-06T20:39:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-09-06T20:39:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-09-06T20:39:26,871 INFO [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-09-06T20:39:26,871 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-09-06T20:39:26,872 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-09-06T20:39:26,872 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-09-06T20:39:26,873 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:39:26,874 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:26,874 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:39:26,874 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:39:26,874 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-09-06T20:39:26,875 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-09-06T20:39:26,875 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-09-06T20:39:26,875 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-09-06T20:39:26,875 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-09-06T20:39:26,875 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-09-06T20:39:26,876 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-09-06T20:39:26,876 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-09-06T20:39:26,876 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-09-06T20:39:26,876 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-09-06T20:39:26,877 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-09-06T20:39:26,877 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-09-06T20:39:26,877 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,45559,1694032716172' joining acquired barrier for procedure (rolllog) in zk 2023-09-06T20:39:26,879 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-09-06T20:39:26,879 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:26,879 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:39:26,879 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:39:26,880 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-09-06T20:39:26,880 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:39:26,880 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:39:26,880 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:39:26,881 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:26,881 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:39:26,881 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,45559,1694032716172' joining acquired barrier for procedure 'rolllog' on coordinator 2023-09-06T20:39:26,881 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@235044b[Count = 0] remaining members to acquire global barrier 2023-09-06T20:39:26,881 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-09-06T20:39:26,881 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-09-06T20:39:26,883 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-09-06T20:39:26,883 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-09-06T20:39:26,883 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-09-06T20:39:26,883 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-09-06T20:39:26,884 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-09-06T20:39:26,884 DEBUG [rs(jenkins-hbase3.apache.org,45559,1694032716172)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:26,884 INFO [rs(jenkins-hbase3.apache.org,45559,1694032716172)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1694032755742 highest: 1694032755742 on jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:26,884 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45559%2C1694032716172.meta:.meta(num 1694032755722) roll requested 2023-09-06T20:39:26,895 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.meta.1694032766884.meta, exclude list is [], retry=0 2023-09-06T20:39:26,899 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41237,DS-07aa54ab-1cb2-4831-8bde-a715b49e632a,DISK] 2023-09-06T20:39:26,901 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.meta.1694032755722.meta with entries=10, filesize=2.50 KB; new WAL /user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.meta.1694032766884.meta 2023-09-06T20:39:26,901 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41237,DS-07aa54ab-1cb2-4831-8bde-a715b49e632a,DISK]] 2023-09-06T20:39:26,901 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.meta.1694032755722.meta is not closed yet, will try archiving it next time 2023-09-06T20:39:26,901 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45559%2C1694032716172:(num 1694032755742) roll requested 2023-09-06T20:39:26,904 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.meta.1694032755722.meta not finished, retry = 0 2023-09-06T20:39:26,913 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.1694032766901, exclude list is [], retry=0 2023-09-06T20:39:26,916 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41237,DS-07aa54ab-1cb2-4831-8bde-a715b49e632a,DISK] 2023-09-06T20:39:26,918 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.1694032755742 with entries=16, filesize=4.15 KB; new WAL /user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.1694032766901 2023-09-06T20:39:26,918 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41237,DS-07aa54ab-1cb2-4831-8bde-a715b49e632a,DISK]] 2023-09-06T20:39:26,918 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.1694032755742 is not closed yet, will try archiving it next time 2023-09-06T20:39:26,920 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/WALs/jenkins-hbase3.apache.org,45559,1694032716172/jenkins-hbase3.apache.org%2C45559%2C1694032716172.1694032748713 to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/oldWALs/jenkins-hbase3.apache.org%2C45559%2C1694032716172.1694032748713 2023-09-06T20:39:26,924 DEBUG [rs(jenkins-hbase3.apache.org,45559,1694032716172)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-09-06T20:39:26,924 INFO [rs(jenkins-hbase3.apache.org,45559,1694032716172)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1694032766901 on jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:26,931 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-09-06T20:39:26,931 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,45559,1694032716172' in zk 2023-09-06T20:39:26,934 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-09-06T20:39:26,934 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:39:26,934 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:26,934 DEBUG [member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-09-06T20:39:26,934 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:39:26,934 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:39:26,935 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:39:26,935 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:39:26,935 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:39:26,936 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:26,936 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:39:26,936 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:39:26,937 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:26,937 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,45559,1694032716172': 2023-09-06T20:39:26,937 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,45559,1694032716172' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-09-06T20:39:26,937 INFO [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-09-06T20:39:26,937 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-09-06T20:39:26,937 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-09-06T20:39:26,937 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-09-06T20:39:26,937 INFO [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-09-06T20:39:26,939 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-09-06T20:39:26,939 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-09-06T20:39:26,939 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-09-06T20:39:26,939 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:39:26,939 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-09-06T20:39:26,940 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-09-06T20:39:26,940 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:39:26,940 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:39:26,940 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:39:26,940 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-09-06T20:39:26,940 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-09-06T20:39:26,941 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:39:26,941 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:26,941 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-09-06T20:39:26,942 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-09-06T20:39:26,942 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:26,945 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-09-06T20:39:26,945 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-09-06T20:39:26,945 DEBUG [(jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-09-06T20:39:26,945 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-09-06T20:39:26,946 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-09-06T20:39:26,946 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-09-06T20:39:26,946 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-09-06T20:39:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-09-06T20:39:27,049 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1694032767049_default_test-1694032725153 table=test-1694032725153 type=FLUSH ttl=0 } 2023-09-06T20:39:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032767049 (current time:1694032767049). 2023-09-06T20:39:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:39:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1694032767049_default_test-1694032725153 VERSION not specified, setting to 2 2023-09-06T20:39:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e979e15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:39:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:39:27,062 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:33532, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:39:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:39:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:39:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=78, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=78, snapshot={ ss=snapshot_1694032767049_default_test-1694032725153 table=test-1694032725153 type=FLUSH ttl=0 } 2023-09-06T20:39:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1694032767049_default_test-1694032725153 table=test-1694032725153 type=FLUSH ttl=0 }, snapshot procedure id = 78 2023-09-06T20:39:27,066 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=78, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=78, snapshot={ ss=snapshot_1694032767049_default_test-1694032725153 table=test-1694032725153 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:39:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=78 2023-09-06T20:39:27,067 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=78, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=78, snapshot={ ss=snapshot_1694032767049_default_test-1694032725153 table=test-1694032725153 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:39:27,069 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=78, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=78, snapshot={ ss=snapshot_1694032767049_default_test-1694032725153 table=test-1694032725153 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:39:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=78 2023-09-06T20:39:27,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=78 2023-09-06T20:39:27,475 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=78, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=78, snapshot={ ss=snapshot_1694032767049_default_test-1694032725153 table=test-1694032725153 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:39:27,475 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE; SnapshotRegionProcedure 7499612530301bea74e2d6f06f601777}] 2023-09-06T20:39:27,477 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=79, ppid=78, state=RUNNABLE; SnapshotRegionProcedure 7499612530301bea74e2d6f06f601777 2023-09-06T20:39:27,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45559 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=79 2023-09-06T20:39:27,629 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=79}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1694032725153,,1694032725664.7499612530301bea74e2d6f06f601777. 2023-09-06T20:39:27,629 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=79}] regionserver.HRegion(2520): Flush status journal for 7499612530301bea74e2d6f06f601777: 2023-09-06T20:39:27,629 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=79}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1694032725153,,1694032725664.7499612530301bea74e2d6f06f601777. for snapshot_1694032767049_default_test-1694032725153 completed. 2023-09-06T20:39:27,629 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=79}] snapshot.SnapshotManifest(240): Storing 'test-1694032725153,,1694032725664.7499612530301bea74e2d6f06f601777.' region-info for snapshot=snapshot_1694032767049_default_test-1694032725153 2023-09-06T20:39:27,629 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=79}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:39:27,629 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=79}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/default/test-1694032725153/7499612530301bea74e2d6f06f601777/f/fb7f27fc980f49db93fa2308b21c456f] hfiles 2023-09-06T20:39:27,629 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=79}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/default/test-1694032725153/7499612530301bea74e2d6f06f601777/f/fb7f27fc980f49db93fa2308b21c456f for snapshot=snapshot_1694032767049_default_test-1694032725153 2023-09-06T20:39:27,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=78 2023-09-06T20:39:28,035 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=79}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1694032725153,,1694032725664.7499612530301bea74e2d6f06f601777. 2023-09-06T20:39:28,036 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=79 2023-09-06T20:39:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(3995): Remote procedure done, pid=79 2023-09-06T20:39:28,036 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1694032767049_default_test-1694032725153 on region 7499612530301bea74e2d6f06f601777 2023-09-06T20:39:28,037 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=79, ppid=78, state=RUNNABLE; SnapshotRegionProcedure 7499612530301bea74e2d6f06f601777 2023-09-06T20:39:28,039 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=79, resume processing ppid=78 2023-09-06T20:39:28,039 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=78, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=78, snapshot={ ss=snapshot_1694032767049_default_test-1694032725153 table=test-1694032725153 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:39:28,040 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=79, ppid=78, state=SUCCESS; SnapshotRegionProcedure 7499612530301bea74e2d6f06f601777 in 563 msec 2023-09-06T20:39:28,040 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=78, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=78, snapshot={ ss=snapshot_1694032767049_default_test-1694032725153 table=test-1694032725153 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:39:28,040 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=78, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=78, snapshot={ ss=snapshot_1694032767049_default_test-1694032725153 table=test-1694032725153 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:39:28,040 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1694032767049_default_test-1694032725153 2023-09-06T20:39:28,041 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/.tmp/snapshot_1694032767049_default_test-1694032725153 2023-09-06T20:39:28,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=78 2023-09-06T20:39:28,452 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=78, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=78, snapshot={ ss=snapshot_1694032767049_default_test-1694032725153 table=test-1694032725153 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:39:28,457 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=78, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=78, snapshot={ ss=snapshot_1694032767049_default_test-1694032725153 table=test-1694032725153 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:39:28,458 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/.tmp/snapshot_1694032767049_default_test-1694032725153 to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/snapshot_1694032767049_default_test-1694032725153 2023-09-06T20:39:28,460 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=78, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=78, snapshot={ ss=snapshot_1694032767049_default_test-1694032725153 table=test-1694032725153 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:39:28,460 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1694032767049_default_test-1694032725153 table=test-1694032725153 type=FLUSH ttl=0 }, snapshot procedure id = 78 2023-09-06T20:39:28,461 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=78, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=78, snapshot={ ss=snapshot_1694032767049_default_test-1694032725153 table=test-1694032725153 type=FLUSH ttl=0 } in 1.3960 sec 2023-09-06T20:39:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=78 2023-09-06T20:39:29,173 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1694032769173_ns2_test-16940327251531 table=ns2:test-16940327251531 type=FLUSH ttl=0 } 2023-09-06T20:39:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032769173 (current time:1694032769173). 2023-09-06T20:39:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:39:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1694032769173_ns2_test-16940327251531 VERSION not specified, setting to 2 2023-09-06T20:39:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1efc68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:39:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:39:29,185 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:33534, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:39:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:39:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:39:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=80, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=80, snapshot={ ss=snapshot_1694032769173_ns2_test-16940327251531 table=ns2:test-16940327251531 type=FLUSH ttl=0 } 2023-09-06T20:39:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1694032769173_ns2_test-16940327251531 table=ns2:test-16940327251531 type=FLUSH ttl=0 }, snapshot procedure id = 80 2023-09-06T20:39:29,189 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=80, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=80, snapshot={ ss=snapshot_1694032769173_ns2_test-16940327251531 table=ns2:test-16940327251531 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:39:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=80 2023-09-06T20:39:29,189 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=80, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=80, snapshot={ ss=snapshot_1694032769173_ns2_test-16940327251531 table=ns2:test-16940327251531 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:39:29,191 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=80, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=80, snapshot={ ss=snapshot_1694032769173_ns2_test-16940327251531 table=ns2:test-16940327251531 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:39:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=80 2023-09-06T20:39:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=80 2023-09-06T20:39:29,597 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=80, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=80, snapshot={ ss=snapshot_1694032769173_ns2_test-16940327251531 table=ns2:test-16940327251531 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-09-06T20:39:29,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE; SnapshotRegionProcedure 87e84cabad1c9957eead562b206d922d}] 2023-09-06T20:39:29,599 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=81, ppid=80, state=RUNNABLE; SnapshotRegionProcedure 87e84cabad1c9957eead562b206d922d 2023-09-06T20:39:29,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45559 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=81 2023-09-06T20:39:29,751 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=81}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16940327251531,,1694032727923.87e84cabad1c9957eead562b206d922d. 2023-09-06T20:39:29,751 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=81}] regionserver.HRegion(2520): Flush status journal for 87e84cabad1c9957eead562b206d922d: 2023-09-06T20:39:29,751 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=81}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16940327251531,,1694032727923.87e84cabad1c9957eead562b206d922d. for snapshot_1694032769173_ns2_test-16940327251531 completed. 2023-09-06T20:39:29,751 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=81}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16940327251531,,1694032727923.87e84cabad1c9957eead562b206d922d.' region-info for snapshot=snapshot_1694032769173_ns2_test-16940327251531 2023-09-06T20:39:29,751 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=81}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-09-06T20:39:29,751 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=81}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/ns2/test-16940327251531/87e84cabad1c9957eead562b206d922d/f/e976e7f8d3c44b73991e878da4fa6496] hfiles 2023-09-06T20:39:29,751 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=81}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/ns2/test-16940327251531/87e84cabad1c9957eead562b206d922d/f/e976e7f8d3c44b73991e878da4fa6496 for snapshot=snapshot_1694032769173_ns2_test-16940327251531 2023-09-06T20:39:29,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=80 2023-09-06T20:39:30,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=81}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16940327251531,,1694032727923.87e84cabad1c9957eead562b206d922d. 2023-09-06T20:39:30,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=81 2023-09-06T20:39:30,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster(3995): Remote procedure done, pid=81 2023-09-06T20:39:30,158 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1694032769173_ns2_test-16940327251531 on region 87e84cabad1c9957eead562b206d922d 2023-09-06T20:39:30,159 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=81, ppid=80, state=RUNNABLE; SnapshotRegionProcedure 87e84cabad1c9957eead562b206d922d 2023-09-06T20:39:30,161 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=81, resume processing ppid=80 2023-09-06T20:39:30,161 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=80, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=80, snapshot={ ss=snapshot_1694032769173_ns2_test-16940327251531 table=ns2:test-16940327251531 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-09-06T20:39:30,161 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=81, ppid=80, state=SUCCESS; SnapshotRegionProcedure 87e84cabad1c9957eead562b206d922d in 563 msec 2023-09-06T20:39:30,162 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=80, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=80, snapshot={ ss=snapshot_1694032769173_ns2_test-16940327251531 table=ns2:test-16940327251531 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:39:30,162 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=80, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=80, snapshot={ ss=snapshot_1694032769173_ns2_test-16940327251531 table=ns2:test-16940327251531 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:39:30,163 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1694032769173_ns2_test-16940327251531 2023-09-06T20:39:30,163 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/.tmp/snapshot_1694032769173_ns2_test-16940327251531 2023-09-06T20:39:30,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=80 2023-09-06T20:39:30,573 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=80, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=80, snapshot={ ss=snapshot_1694032769173_ns2_test-16940327251531 table=ns2:test-16940327251531 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:39:30,579 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=80, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=80, snapshot={ ss=snapshot_1694032769173_ns2_test-16940327251531 table=ns2:test-16940327251531 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:39:30,579 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/.tmp/snapshot_1694032769173_ns2_test-16940327251531 to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/snapshot_1694032769173_ns2_test-16940327251531 2023-09-06T20:39:30,580 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=80, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=80, snapshot={ ss=snapshot_1694032769173_ns2_test-16940327251531 table=ns2:test-16940327251531 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:39:30,580 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1694032769173_ns2_test-16940327251531 table=ns2:test-16940327251531 type=FLUSH ttl=0 }, snapshot procedure id = 80 2023-09-06T20:39:30,581 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=80, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=80, snapshot={ ss=snapshot_1694032769173_ns2_test-16940327251531 table=ns2:test-16940327251531 type=FLUSH ttl=0 } in 1.3930 sec 2023-09-06T20:39:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=80 2023-09-06T20:39:31,295 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] backup.TestBackupBase$FullTableBackupClientForTest(226): snapshot copy for backup_1694032764728 2023-09-06T20:39:31,295 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-09-06T20:39:31,297 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-09-06T20:39:31,297 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1694032764728_ns2:test-16940327251531 2023-09-06T20:39:31,297 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1694032769173_ns2_test-16940327251531 to hdfs://localhost:34451/backupUT/backup_1694032764728/ns2/test-16940327251531/ 2023-09-06T20:39:31,301 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-09-06T20:39:31,327 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34451, inputRoot=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0 2023-09-06T20:39:31,327 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,41705,1694032713168_328487747_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34451/backupUT/backup_1694032764728/ns2/test-16940327251531, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34451/backupUT/backup_1694032764728/ns2/test-16940327251531/.hbase-snapshot/.tmp/snapshot_1694032769173_ns2_test-16940327251531 2023-09-06T20:39:31,327 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34451, inputRoot=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0, snapshotDir=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/snapshot_1694032769173_ns2_test-16940327251531. 2023-09-06T20:39:31,338 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/snapshot_1694032769173_ns2_test-16940327251531 to hdfs://localhost:34451/backupUT/backup_1694032764728/ns2/test-16940327251531/.hbase-snapshot/.tmp/snapshot_1694032769173_ns2_test-16940327251531 2023-09-06T20:39:31,763 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:39:31,763 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:39:31,764 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:39:31,764 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:39:32,752 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/hadoop-8063571004480693553.jar 2023-09-06T20:39:32,753 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:39:32,753 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:39:32,755 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:39:32,755 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:39:32,756 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:39:32,756 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:39:32,756 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:39:32,757 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:39:32,757 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:39:32,758 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:39:32,759 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:39:32,759 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:39:32,760 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:39:32,760 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:39:32,760 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:39:32,761 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:39:32,761 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:39:32,762 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:39:32,762 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:39:32,763 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:39:32,763 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:39:32,767 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:39:32,768 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:39:32,768 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:39:32,768 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:39:32,768 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:39:32,769 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:39:32,769 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:39:32,851 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1694032769173_ns2_test-16940327251531' hfile list 2023-09-06T20:39:32,860 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-09-06T20:39:33,078 WARN [Listener at jenkins-hbase3.apache.org/41977 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/mapreduce.cluster.local.dir/1694032773029/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:39:33,078 WARN [Listener at jenkins-hbase3.apache.org/41977 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/mapreduce.cluster.local.dir/1694032773029/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:39:33,242 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-09-06T20:39:33,296 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/ns2/test-16940327251531/87e84cabad1c9957eead562b206d922d/f/e976e7f8d3c44b73991e878da4fa6496 output=hdfs://localhost:34451/backupUT/backup_1694032764728/ns2/test-16940327251531/archive/data/ns2/test-16940327251531/87e84cabad1c9957eead562b206d922d/f/e976e7f8d3c44b73991e878da4fa6496 2023-09-06T20:39:33,297 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-09-06T20:39:34,154 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-09-06T20:39:34,155 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-09-06T20:39:34,162 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1694032769173_ns2_test-16940327251531 2023-09-06T20:39:34,163 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1694032769173_ns2_test-16940327251531 finished. 2023-09-06T20:39:34,163 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1694032764728_test-1694032725153 2023-09-06T20:39:34,163 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1694032767049_default_test-1694032725153 to hdfs://localhost:34451/backupUT/backup_1694032764728/default/test-1694032725153/ 2023-09-06T20:39:34,163 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-09-06T20:39:34,191 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34451, inputRoot=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0 2023-09-06T20:39:34,191 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,41705,1694032713168_-1018582152_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34451/backupUT/backup_1694032764728/default/test-1694032725153, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34451/backupUT/backup_1694032764728/default/test-1694032725153/.hbase-snapshot/.tmp/snapshot_1694032767049_default_test-1694032725153 2023-09-06T20:39:34,191 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34451, inputRoot=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0, snapshotDir=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/snapshot_1694032767049_default_test-1694032725153. 2023-09-06T20:39:34,201 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/snapshot_1694032767049_default_test-1694032725153 to hdfs://localhost:34451/backupUT/backup_1694032764728/default/test-1694032725153/.hbase-snapshot/.tmp/snapshot_1694032767049_default_test-1694032725153 2023-09-06T20:39:34,270 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-09-06T20:39:34,270 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-09-06T20:39:34,623 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-09-06T20:39:34,623 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-09-06T20:39:34,624 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-09-06T20:39:34,625 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-09-06T20:39:35,694 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/hadoop-8391618237581185519.jar 2023-09-06T20:39:35,694 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:39:35,695 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-09-06T20:39:35,695 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-09-06T20:39:35,695 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-09-06T20:39:35,696 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-09-06T20:39:35,696 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-09-06T20:39:35,697 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-09-06T20:39:35,697 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-09-06T20:39:35,697 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-09-06T20:39:35,698 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-09-06T20:39:35,699 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-09-06T20:39:35,699 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-09-06T20:39:35,700 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-09-06T20:39:35,700 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-09-06T20:39:35,701 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-09-06T20:39:35,701 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-09-06T20:39:35,701 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-09-06T20:39:35,702 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-09-06T20:39:35,702 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-09-06T20:39:35,703 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-09-06T20:39:35,703 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-09-06T20:39:35,704 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:39:35,704 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:39:35,705 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:39:35,706 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:39:35,706 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-09-06T20:39:35,706 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:39:35,707 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-09-06T20:39:35,754 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1694032767049_default_test-1694032725153' hfile list 2023-09-06T20:39:35,756 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-09-06T20:39:35,830 WARN [Listener at jenkins-hbase3.apache.org/41977 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/mapreduce.cluster.local.dir/1694032775795/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-09-06T20:39:35,830 WARN [Listener at jenkins-hbase3.apache.org/41977 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/mapreduce.cluster.local.dir/1694032775795/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-09-06T20:39:35,887 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-09-06T20:39:35,911 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/default/test-1694032725153/7499612530301bea74e2d6f06f601777/f/fb7f27fc980f49db93fa2308b21c456f output=hdfs://localhost:34451/backupUT/backup_1694032764728/default/test-1694032725153/archive/data/default/test-1694032725153/7499612530301bea74e2d6f06f601777/f/fb7f27fc980f49db93fa2308b21c456f 2023-09-06T20:39:35,911 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 4.004M/sec 2023-09-06T20:39:36,871 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-09-06T20:39:36,872 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-09-06T20:39:36,879 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1694032767049_default_test-1694032725153 2023-09-06T20:39:36,879 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1694032767049_default_test-1694032725153 finished. 2023-09-06T20:39:36,880 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(1014): ns2:test-16940327251531 2023-09-06T20:39:36,880 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(1014): test-1694032725153 2023-09-06T20:39:36,896 ERROR [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(228): Unexpected BackupException : Failed stage stage_4 in testing java.io.IOException: Failed stage stage_4 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:245) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.runBackupAndFailAtStage(TestFullBackupWithFailures.java:67) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.testFullBackupWithFailures(TestFullBackupWithFailures.java:55) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-09-06T20:39:36,896 ERROR [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(240): BackupId=backup_1694032764728,startts=1694032766866,failedts=1694032776896,failedphase=SNAPSHOTCOPY,failedmessage=Failed stage stage_4 in testing 2023-09-06T20:39:36,896 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-09-06T20:39:36,896 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1694032769173_ns2_test-16940327251531 2023-09-06T20:39:36,897 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1694032769173_ns2_test-16940327251531" 2023-09-06T20:39:36,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1694032769173_ns2_test-16940327251531 2023-09-06T20:39:36,900 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1694032769173_ns2_test-16940327251531 for backup backup_1694032764728 succeeded. 2023-09-06T20:39:36,900 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1694032767049_default_test-1694032725153 2023-09-06T20:39:36,901 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1694032767049_default_test-1694032725153" 2023-09-06T20:39:36,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1694032767049_default_test-1694032725153 2023-09-06T20:39:36,905 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1694032767049_default_test-1694032725153 for backup backup_1694032764728 succeeded. 2023-09-06T20:39:36,905 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-09-06T20:39:36,914 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable backup:system 2023-09-06T20:39:36,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-09-06T20:39:36,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=82 2023-09-06T20:39:36,919 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032776919"}]},"ts":"1694032776919"} 2023-09-06T20:39:36,920 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-09-06T20:39:36,922 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-09-06T20:39:36,922 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, UNASSIGN}] 2023-09-06T20:39:36,924 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=83, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, UNASSIGN 2023-09-06T20:39:36,925 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=83 updating hbase:meta row=de261a559c0224b9046b4ead35ab8e78, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:36,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE; CloseRegionProcedure de261a559c0224b9046b4ead35ab8e78, server=jenkins-hbase3.apache.org,45559,1694032716172}] 2023-09-06T20:39:37,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=82 2023-09-06T20:39:37,077 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] handler.UnassignRegionHandler(119): Close de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:37,077 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HRegion(1668): Closing de261a559c0224b9046b4ead35ab8e78, disabling compactions & flushes 2023-09-06T20:39:37,077 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HRegion(1690): Closing region backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:37,077 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:37,077 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. after waiting 0 ms 2023-09-06T20:39:37,077 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:37,078 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HRegion(2819): Flushing de261a559c0224b9046b4ead35ab8e78 2/2 column families, dataSize=999 B heapSize=2.05 KB 2023-09-06T20:39:37,091 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/meta/75d80de97d0d44b399e21858a20e3dad is 171, key is trslm:hdfs://localhost:34451/backupUT\x00ns2:test-16940327251531/meta:log-roll-map/1694032776892/Put/seqid=0 2023-09-06T20:39:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=82 2023-09-06T20:39:37,496 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=785 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/meta/75d80de97d0d44b399e21858a20e3dad 2023-09-06T20:39:37,504 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/4c5430942f084a408de15dbdba0b6c83 is 218, key is session:backup_1694032764728/session:context/1694032766867/Put/seqid=0 2023-09-06T20:39:37,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=82 2023-09-06T20:39:37,909 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=214 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/4c5430942f084a408de15dbdba0b6c83 2023-09-06T20:39:37,916 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/meta/75d80de97d0d44b399e21858a20e3dad as hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/meta/75d80de97d0d44b399e21858a20e3dad 2023-09-06T20:39:37,921 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/meta/75d80de97d0d44b399e21858a20e3dad, entries=6, sequenceid=72, filesize=5.8 K 2023-09-06T20:39:37,922 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/4c5430942f084a408de15dbdba0b6c83 as hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/4c5430942f084a408de15dbdba0b6c83 2023-09-06T20:39:37,928 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/4c5430942f084a408de15dbdba0b6c83, entries=1, sequenceid=72, filesize=5.2 K 2023-09-06T20:39:37,929 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HRegion(3022): Finished flush of dataSize ~999 B/999, heapSize ~2.02 KB/2072, currentSize=0 B/0 for de261a559c0224b9046b4ead35ab8e78 in 851ms, sequenceid=72, compaction requested=true 2023-09-06T20:39:37,941 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/recovered.edits/75.seqid, newMaxSeqId=75, maxSeqId=57 2023-09-06T20:39:37,941 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:39:37,942 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HRegion(1909): Closed backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:37,942 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HRegion(1622): Region close journal for de261a559c0224b9046b4ead35ab8e78: 2023-09-06T20:39:37,943 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] handler.UnassignRegionHandler(163): Closed de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:37,944 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=83 updating hbase:meta row=de261a559c0224b9046b4ead35ab8e78, regionState=CLOSED 2023-09-06T20:39:37,947 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=84, resume processing ppid=83 2023-09-06T20:39:37,947 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=84, ppid=83, state=SUCCESS; CloseRegionProcedure de261a559c0224b9046b4ead35ab8e78, server=jenkins-hbase3.apache.org,45559,1694032716172 in 1.0190 sec 2023-09-06T20:39:37,948 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=83, resume processing ppid=82 2023-09-06T20:39:37,949 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=83, ppid=82, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, UNASSIGN in 1.0250 sec 2023-09-06T20:39:37,949 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032777949"}]},"ts":"1694032777949"} 2023-09-06T20:39:37,951 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-09-06T20:39:37,955 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-09-06T20:39:37,957 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=82, state=SUCCESS; DisableTableProcedure table=backup:system in 1.0420 sec 2023-09-06T20:39:38,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=82 2023-09-06T20:39:38,031 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1694032778030 table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:39:38,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1694032778031 (current time:1694032778031). 2023-09-06T20:39:38,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-09-06T20:39:38,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1694032778030 VERSION not specified, setting to 2 2023-09-06T20:39:38,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cba8d9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-09-06T20:39:38,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-09-06T20:39:38,040 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:59644, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-09-06T20:39:38,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:39:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-09-06T20:39:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=85, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=85, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032778030 table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:39:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032778030 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 85 2023-09-06T20:39:38,044 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=85, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=85, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032778030 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-09-06T20:39:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=85 2023-09-06T20:39:38,045 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=85, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=85, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032778030 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-09-06T20:39:38,047 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=85, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=85, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032778030 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-09-06T20:39:38,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=85 2023-09-06T20:39:38,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=85 2023-09-06T20:39:38,454 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=85, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=85, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032778030 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-09-06T20:39:38,455 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-09-06T20:39:38,455 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-09-06T20:39:38,455 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:39:38,456 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/meta/75d80de97d0d44b399e21858a20e3dad] hfiles 2023-09-06T20:39:38,456 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/1): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/meta/75d80de97d0d44b399e21858a20e3dad 2023-09-06T20:39:38,456 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:39:38,458 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/21801e1055db41adac61df285d558318, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/4c5430942f084a408de15dbdba0b6c83, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/9585b49cb83643a185229f157af65936, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/98a5e9c6fde5479191b554626ed8f3c4] hfiles 2023-09-06T20:39:38,458 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/4): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/21801e1055db41adac61df285d558318 2023-09-06T20:39:38,458 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/4): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/4c5430942f084a408de15dbdba0b6c83 2023-09-06T20:39:38,458 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/4): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/9585b49cb83643a185229f157af65936 2023-09-06T20:39:38,458 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (4/4): hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/98a5e9c6fde5479191b554626ed8f3c4 2023-09-06T20:39:38,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=85 2023-09-06T20:39:38,864 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => de261a559c0224b9046b4ead35ab8e78, NAME => 'backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-09-06T20:39:38,865 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=85, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=85, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032778030 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-09-06T20:39:38,865 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=85, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=85, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032778030 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-09-06T20:39:38,865 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1694032778030 2023-09-06T20:39:38,866 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1694032778030 2023-09-06T20:39:39,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=85 2023-09-06T20:39:39,276 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=85, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=85, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032778030 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-09-06T20:39:39,285 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=85, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=85, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032778030 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-09-06T20:39:39,286 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1694032778030 to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1694032778030 2023-09-06T20:39:39,287 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=85, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=85, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032778030 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-09-06T20:39:39,287 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032778030 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 85 2023-09-06T20:39:39,288 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=85, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=85, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1694032778030 table=backup:system type=FLUSH ttl=0 } in 1.2450 sec 2023-09-06T20:39:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=85 2023-09-06T20:39:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=86, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032764745 type: FLUSH version: 2 ttl: 0 ) 2023-09-06T20:39:40,158 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(1015): Restore snapshot=snapshot_backup_system as table=backup:system 2023-09-06T20:39:40,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=86 2023-09-06T20:39:40,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=86 2023-09-06T20:39:40,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=86 2023-09-06T20:39:40,546 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 42b03c0f5b2d2278a9b485cc44396dc1 changed from -1.0 to 0.0, refreshing cache 2023-09-06T20:39:40,571 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/.tabledesc/.tableinfo.0000000005.635 2023-09-06T20:39:40,571 INFO [PEWorker-1 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/.tabledesc/.tableinfo.0000000006.635 2023-09-06T20:39:40,573 INFO [PEWorker-1 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-09-06T20:39:40,575 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032764745 type: FLUSH version: 2 ttl: 0 2023-09-06T20:39:40,575 DEBUG [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system 2023-09-06T20:39:40,577 DEBUG [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-09-06T20:39:40,578 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(227): region to restore: de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:40,583 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/meta/75d80de97d0d44b399e21858a20e3dad to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/archive/data/backup/system/de261a559c0224b9046b4ead35ab8e78/meta/75d80de97d0d44b399e21858a20e3dad 2023-09-06T20:39:40,583 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:39:40,587 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/4c5430942f084a408de15dbdba0b6c83 to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/archive/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/4c5430942f084a408de15dbdba0b6c83 2023-09-06T20:39:40,587 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:39:40,587 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032764745 type: FLUSH version: 2 ttl: 0 2023-09-06T20:39:40,588 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1694032780588"}]},"ts":"9223372036854775807"} 2023-09-06T20:39:40,590 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-09-06T20:39:40,591 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => de261a559c0224b9046b4ead35ab8e78, NAME => 'backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:39:40,591 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1694032780589"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1694032780589"}]},"ts":"1694032780589"} 2023-09-06T20:39:40,592 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-09-06T20:39:40,592 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-09-06T20:39:40,592 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => de261a559c0224b9046b4ead35ab8e78, NAME => 'backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.', STARTKEY => '', ENDKEY => ''}] 2023-09-06T20:39:40,592 INFO [PEWorker-1 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-09-06T20:39:40,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=86, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1694032764745 type: FLUSH version: 2 ttl: 0 ) in 435 msec 2023-09-06T20:39:40,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=86 2023-09-06T20:39:40,765 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1694032778030" 2023-09-06T20:39:40,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(378): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1694032778030 2023-09-06T20:39:40,769 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.HMaster$11(2631): Client=jenkins//172.31.12.81 enable backup:system 2023-09-06T20:39:40,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] procedure2.ProcedureExecutor(1032): Stored pid=87, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-09-06T20:39:40,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=87 2023-09-06T20:39:40,772 INFO [PEWorker-4 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-09-06T20:39:40,773 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032780773"}]},"ts":"1694032780773"} 2023-09-06T20:39:40,774 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-09-06T20:39:40,776 DEBUG [PEWorker-4 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-09-06T20:39:40,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, ASSIGN}] 2023-09-06T20:39:40,778 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, ASSIGN 2023-09-06T20:39:40,778 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-09-06T20:39:40,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=87 2023-09-06T20:39:40,930 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=88 updating hbase:meta row=de261a559c0224b9046b4ead35ab8e78, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:40,931 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE; OpenRegionProcedure de261a559c0224b9046b4ead35ab8e78, server=jenkins-hbase3.apache.org,45559,1694032716172}] 2023-09-06T20:39:41,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=87 2023-09-06T20:39:41,089 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(130): Open backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:41,089 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7254): Opening region: {ENCODED => de261a559c0224b9046b4ead35ab8e78, NAME => 'backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.', STARTKEY => '', ENDKEY => ''} 2023-09-06T20:39:41,090 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-09-06T20:39:41,090 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:41,090 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(888): Instantiated backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-09-06T20:39:41,090 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7296): checking encryption for de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:41,090 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7299): checking classloading for de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:41,092 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:41,093 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de261a559c0224b9046b4ead35ab8e78 columnFamilyName meta 2023-09-06T20:39:41,093 DEBUG [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:39:41,093 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.HStore(324): Store=de261a559c0224b9046b4ead35ab8e78/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:39:41,094 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:41,094 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de261a559c0224b9046b4ead35ab8e78 columnFamilyName session 2023-09-06T20:39:41,094 DEBUG [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-09-06T20:39:41,101 DEBUG [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/21801e1055db41adac61df285d558318 2023-09-06T20:39:41,105 DEBUG [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/9585b49cb83643a185229f157af65936 2023-09-06T20:39:41,110 DEBUG [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/98a5e9c6fde5479191b554626ed8f3c4 2023-09-06T20:39:41,110 INFO [StoreOpener-de261a559c0224b9046b4ead35ab8e78-1 {}] regionserver.HStore(324): Store=de261a559c0224b9046b4ead35ab8e78/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-06T20:39:41,111 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:41,112 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:41,114 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-09-06T20:39:41,116 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1076): writing seq id for de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:41,117 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): Opened de261a559c0224b9046b4ead35ab8e78; next sequenceid=76; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9447976480, jitterRate=-0.1200886219739914}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-09-06T20:39:41,118 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(992): Region open journal for de261a559c0224b9046b4ead35ab8e78: 2023-09-06T20:39:41,118 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78., pid=89, masterSystemTime=1694032781083 2023-09-06T20:39:41,119 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.CompactSplit(403): Add compact mark for store de261a559c0224b9046b4ead35ab8e78:session, priority=-2147483648, current under compaction store size is 1 2023-09-06T20:39:41,119 DEBUG [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-09-06T20:39:41,119 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:39:41,121 DEBUG [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 16482 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-09-06T20:39:41,121 DEBUG [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] regionserver.HStore(1534): de261a559c0224b9046b4ead35ab8e78/session is initiating minor compaction (all files) 2023-09-06T20:39:41,121 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:41,121 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(158): Opened backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:41,121 INFO [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of de261a559c0224b9046b4ead35ab8e78/session in backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:41,121 INFO [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/21801e1055db41adac61df285d558318, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/9585b49cb83643a185229f157af65936, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/98a5e9c6fde5479191b554626ed8f3c4] into tmpdir=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp, totalSize=16.1 K 2023-09-06T20:39:41,122 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=88 updating hbase:meta row=de261a559c0224b9046b4ead35ab8e78, regionState=OPEN, openSeqNum=76, regionLocation=jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:41,122 DEBUG [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] compactions.Compactor(223): Compacting 21801e1055db41adac61df285d558318, keycount=3, bloomtype=ROW, size=5.6 K, encoding=NONE, compression=NONE, seqNum=32, earliestPutTs=1694032742622 2023-09-06T20:39:41,122 DEBUG [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] compactions.Compactor(223): Compacting 9585b49cb83643a185229f157af65936, keycount=2, bloomtype=ROW, size=5.2 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1694032753417 2023-09-06T20:39:41,123 DEBUG [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] compactions.Compactor(223): Compacting 98a5e9c6fde5479191b554626ed8f3c4, keycount=2, bloomtype=ROW, size=5.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1694032764630 2023-09-06T20:39:41,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=89, resume processing ppid=88 2023-09-06T20:39:41,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=89, ppid=88, state=SUCCESS; OpenRegionProcedure de261a559c0224b9046b4ead35ab8e78, server=jenkins-hbase3.apache.org,45559,1694032716172 in 192 msec 2023-09-06T20:39:41,126 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=88, resume processing ppid=87 2023-09-06T20:39:41,126 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=88, ppid=87, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=de261a559c0224b9046b4ead35ab8e78, ASSIGN in 348 msec 2023-09-06T20:39:41,127 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1694032781127"}]},"ts":"1694032781127"} 2023-09-06T20:39:41,128 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-09-06T20:39:41,131 INFO [PEWorker-4 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-09-06T20:39:41,134 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=87, state=SUCCESS; EnableTableProcedure table=backup:system in 363 msec 2023-09-06T20:39:41,149 INFO [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de261a559c0224b9046b4ead35ab8e78#session#compaction#17 average throughput is 1.10 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-09-06T20:39:41,150 DEBUG [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/b887480e39dc4bb6863cc7c0cf3bf7c3 is 355, key is session:backup_1694032753545/session:context/1694032764630/Put/seqid=0 2023-09-06T20:39:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=87 2023-09-06T20:39:41,375 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(1525): Done restoring backup system table 2023-09-06T20:39:41,375 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-09-06T20:39:41,382 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-09-06T20:39:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-09-06T20:39:41,385 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-09-06T20:39:41,385 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: SNAPSHOTCOPY 2023-09-06T20:39:41,386 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:34451/backupUT/backup_1694032764728/ns2/test-16940327251531 done. 2023-09-06T20:39:41,387 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(208): hdfs://localhost:34451/backupUT/backup_1694032764728/ns2 is empty, remove it. 2023-09-06T20:39:41,388 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:34451/backupUT/backup_1694032764728/default/test-1694032725153 done. 2023-09-06T20:39:41,389 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(208): hdfs://localhost:34451/backupUT/backup_1694032764728/default is empty, remove it. 2023-09-06T20:39:41,390 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-09-06T20:39:41,396 ERROR [Listener at jenkins-hbase3.apache.org/41977 {}] impl.TableBackupClient(246): Backup backup_1694032764728 failed. Backup session finished. Status: FAILURE 2023-09-06T20:39:41,396 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:39:41,396 ERROR [Listener at jenkins-hbase3.apache.org/41977 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_4 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:255) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.runBackupAndFailAtStage(TestFullBackupWithFailures.java:67) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.testFullBackupWithFailures(TestFullBackupWithFailures.java:55) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_4 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:245) ~[test-classes/:?] ... 32 more 2023-09-06T20:39:41,494 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] hbase.ResourceChecker(175): after: backup.TestFullBackupWithFailures#testFullBackupWithFailures Thread=605 (was 584) Potentially hanging thread: rs(jenkins-hbase3.apache.org,45559,1694032716172)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0xd3d4503-shared-pool-30 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x393f348c-shared-pool-36 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:58035@0x587cf26b java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1455) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1485) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$496/2048259679.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0xd3d4503-shared-pool-31 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,45559,1694032716172)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: region-location-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS:0;jenkins-hbase3:45559-shortCompactions-0 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.DFSOutputStream.completeFile(DFSOutputStream.java:940) org.apache.hadoop.hdfs.DFSOutputStream.completeFile(DFSOutputStream.java:880) org.apache.hadoop.hdfs.DFSOutputStream.closeImpl(DFSOutputStream.java:863) org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:818) org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72) org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.finishClose(HFileWriterImpl.java:858) org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.close(HFileWriterImpl.java:679) org.apache.hadoop.hbase.regionserver.StoreFileWriter.close(StoreFileWriter.java:377) org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.commitWriter(DefaultCompactor.java:72) org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.commitWriter(DefaultCompactor.java:40) org.apache.hadoop.hbase.regionserver.compactions.Compactor.compact(Compactor.java:396) org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.compact(DefaultCompactor.java:64) org.apache.hadoop.hbase.regionserver.DefaultStoreEngine$DefaultCompactionContext.compact(DefaultStoreEngine.java:122) org.apache.hadoop.hbase.regionserver.HStore.compact(HStore.java:1176) org.apache.hadoop.hbase.regionserver.HRegion.compact(HRegion.java:2341) org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.doCompaction(CompactSplit.java:667) org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.run(CompactSplit.java:716) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/cluster_b47b59a8-e7b8-2208-bc7e-15cb99b544fd/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-91659908-172.31.12.81-1694032709610:blk_1073741907_1083, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0xd3d4503-shared-pool-35 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x393f348c-shared-pool-33 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-91659908-172.31.12.81-1694032709610:blk_1073741908_1084, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_721442242_16 at /127.0.0.1:33600 [Waiting for operation #31] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x393f348c-shared-pool-35 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0xd3d4503-shared-pool-33 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_64168197_16 at /127.0.0.1:33624 [Receiving block BP-91659908-172.31.12.81-1694032709610:blk_1073741908_1084] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: (jenkins-hbase3.apache.org,41705,1694032713168)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x393f348c-shared-pool-34 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_721442242_16 at /127.0.0.1:41006 [Waiting for operation #6] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:58035@0x587cf26b-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x393f348c-shared-pool-32 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:58035@0x587cf26b-SendThread(127.0.0.1:58035) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:228) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1338) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1261) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: member: 'jenkins-hbase3.apache.org,45559,1694032716172' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_721442242_16 at /127.0.0.1:36302 [Waiting for operation #7] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_721442242_16 at /127.0.0.1:40926 [Waiting for operation #22] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0xd3d4503-shared-pool-34 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataStreamer for file /user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/b887480e39dc4bb6863cc7c0cf3bf7c3 java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:678) Potentially hanging thread: hconnection-0x393f348c-shared-pool-37 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0xd3d4503-shared-pool-32 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/cluster_b47b59a8-e7b8-2208-bc7e-15cb99b544fd/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_64168197_16 at /127.0.0.1:33620 [Receiving block BP-91659908-172.31.12.81-1694032709610:blk_1073741907_1083] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,45559,1694032716172)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=677 (was 660) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=159 (was 152) - SystemLoadAverage LEAK? -, ProcessCount=172 (was 168) - ProcessCount LEAK? -, AvailableMemoryMB=3489 (was 3643) 2023-09-06T20:39:41,494 WARN [Listener at jenkins-hbase3.apache.org/41977 {}] hbase.ResourceChecker(130): Thread=605 is superior to 500 2023-09-06T20:39:41,501 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-09-06T20:39:41,501 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:39:41,502 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-09-06T20:39:41,502 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] util.JVMClusterUtil(257): Found active master hash=374730863, stopped=false 2023-09-06T20:39:41,502 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] master.ServerManager(910): Cluster shutdown requested of master=jenkins-hbase3.apache.org,41705,1694032713168 2023-09-06T20:39:41,504 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] procedure2.ProcedureExecutor(630): Stopping 2023-09-06T20:39:41,509 DEBUG [Listener at jenkins-hbase3.apache.org/41977 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:39:41,510 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,45559,1694032716172' ***** 2023-09-06T20:39:41,510 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] regionserver.HRegionServer(2531): STOPPED: Shutdown requested 2023-09-06T20:39:41,510 INFO [RS:0;jenkins-hbase3:45559 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-09-06T20:39:41,524 INFO [RS:0;jenkins-hbase3:45559 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@6b0a0cdf{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-09-06T20:39:41,530 INFO [RS:0;jenkins-hbase3:45559 {}] server.AbstractConnector(383): Stopped ServerConnector@2ef3cb42{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-09-06T20:39:41,531 INFO [RS:0;jenkins-hbase3:45559 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-09-06T20:39:41,531 INFO [RS:0;jenkins-hbase3:45559 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@5fdc8a81{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-09-06T20:39:41,532 INFO [RS:0;jenkins-hbase3:45559 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@740cdc54{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/hadoop.log.dir/,STOPPED} 2023-09-06T20:39:41,535 INFO [RS:0;jenkins-hbase3:45559 {}] regionserver.HeapMemoryManager(220): Stopping 2023-09-06T20:39:41,535 INFO [RS:0;jenkins-hbase3:45559 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2023-09-06T20:39:41,535 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-09-06T20:39:41,535 INFO [RS:0;jenkins-hbase3:45559 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-09-06T20:39:41,535 INFO [RS:0;jenkins-hbase3:45559 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-09-06T20:39:41,535 INFO [RS:0;jenkins-hbase3:45559 {}] regionserver.HRegionServer(3528): Received CLOSE for b41ddfc19cb7ed42d2519d4bd835b296 2023-09-06T20:39:41,536 INFO [RS:0;jenkins-hbase3:45559 {}] regionserver.HRegionServer(3528): Received CLOSE for bba17f338feef6f7f45485272a6c03ea 2023-09-06T20:39:41,536 INFO [RS:0;jenkins-hbase3:45559 {}] regionserver.HRegionServer(3528): Received CLOSE for 42b03c0f5b2d2278a9b485cc44396dc1 2023-09-06T20:39:41,536 INFO [RS:0;jenkins-hbase3:45559 {}] regionserver.HRegionServer(3528): Received CLOSE for 87e84cabad1c9957eead562b206d922d 2023-09-06T20:39:41,536 INFO [RS:0;jenkins-hbase3:45559 {}] regionserver.HRegionServer(3528): Received CLOSE for 7499612530301bea74e2d6f06f601777 2023-09-06T20:39:41,536 INFO [RS:0;jenkins-hbase3:45559 {}] regionserver.HRegionServer(3528): Received CLOSE for 17f2e8bf9b7a35cfadd958888b7227bd 2023-09-06T20:39:41,536 INFO [RS:0;jenkins-hbase3:45559 {}] regionserver.HRegionServer(3528): Received CLOSE for de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:41,536 INFO [RS:0;jenkins-hbase3:45559 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,45559,1694032716172 2023-09-06T20:39:41,536 DEBUG [RS:0;jenkins-hbase3:45559 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:39:41,536 INFO [RS:0;jenkins-hbase3:45559 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-09-06T20:39:41,537 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing b41ddfc19cb7ed42d2519d4bd835b296, disabling compactions & flushes 2023-09-06T20:39:41,537 INFO [RS:0;jenkins-hbase3:45559 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-09-06T20:39:41,537 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-16940327251532,,1694032729129.b41ddfc19cb7ed42d2519d4bd835b296. 2023-09-06T20:39:41,537 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16940327251532,,1694032729129.b41ddfc19cb7ed42d2519d4bd835b296. 2023-09-06T20:39:41,537 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16940327251532,,1694032729129.b41ddfc19cb7ed42d2519d4bd835b296. after waiting 0 ms 2023-09-06T20:39:41,537 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16940327251532,,1694032729129.b41ddfc19cb7ed42d2519d4bd835b296. 2023-09-06T20:39:41,541 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/ns3/test-16940327251532/b41ddfc19cb7ed42d2519d4bd835b296/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-09-06T20:39:41,541 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:39:41,542 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-16940327251532,,1694032729129.b41ddfc19cb7ed42d2519d4bd835b296. 2023-09-06T20:39:41,542 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for b41ddfc19cb7ed42d2519d4bd835b296: 2023-09-06T20:39:41,542 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-16940327251532,,1694032729129.b41ddfc19cb7ed42d2519d4bd835b296. 2023-09-06T20:39:41,543 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing bba17f338feef6f7f45485272a6c03ea, disabling compactions & flushes 2023-09-06T20:39:41,543 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1694032735690.bba17f338feef6f7f45485272a6c03ea. 2023-09-06T20:39:41,543 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1694032735690.bba17f338feef6f7f45485272a6c03ea. 2023-09-06T20:39:41,543 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1694032735690.bba17f338feef6f7f45485272a6c03ea. after waiting 0 ms 2023-09-06T20:39:41,543 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1694032735690.bba17f338feef6f7f45485272a6c03ea. 2023-09-06T20:39:41,546 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system_bulk/bba17f338feef6f7f45485272a6c03ea/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-09-06T20:39:41,547 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:39:41,548 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1694032735690.bba17f338feef6f7f45485272a6c03ea. 2023-09-06T20:39:41,548 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for bba17f338feef6f7f45485272a6c03ea: 2023-09-06T20:39:41,548 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1694032735690.bba17f338feef6f7f45485272a6c03ea. 2023-09-06T20:39:41,548 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 42b03c0f5b2d2278a9b485cc44396dc1, disabling compactions & flushes 2023-09-06T20:39:41,548 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1694032719521.42b03c0f5b2d2278a9b485cc44396dc1. 2023-09-06T20:39:41,549 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1694032719521.42b03c0f5b2d2278a9b485cc44396dc1. 2023-09-06T20:39:41,549 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1694032719521.42b03c0f5b2d2278a9b485cc44396dc1. after waiting 0 ms 2023-09-06T20:39:41,549 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1694032719521.42b03c0f5b2d2278a9b485cc44396dc1. 2023-09-06T20:39:41,549 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 42b03c0f5b2d2278a9b485cc44396dc1 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-09-06T20:39:41,555 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-09-06T20:39:41,564 DEBUG [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/b887480e39dc4bb6863cc7c0cf3bf7c3 as hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/b887480e39dc4bb6863cc7c0cf3bf7c3 2023-09-06T20:39:41,568 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/hbase/namespace/42b03c0f5b2d2278a9b485cc44396dc1/.tmp/info/a6e03d17f7d04948a154c2756822ea93 is 45, key is default/info:d/1694032720495/Put/seqid=0 2023-09-06T20:39:41,572 DEBUG [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-09-06T20:39:41,572 INFO [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in de261a559c0224b9046b4ead35ab8e78/session of de261a559c0224b9046b4ead35ab8e78 into b887480e39dc4bb6863cc7c0cf3bf7c3(size=6.2 K), total size for store is 6.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-09-06T20:39:41,573 DEBUG [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for de261a559c0224b9046b4ead35ab8e78: 2023-09-06T20:39:41,573 INFO [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78., storeName=de261a559c0224b9046b4ead35ab8e78/session, priority=13, startTime=1694032781118; duration=0sec 2023-09-06T20:39:41,573 DEBUG [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-09-06T20:39:41,573 DEBUG [RS:0;jenkins-hbase3:45559-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de261a559c0224b9046b4ead35ab8e78:session 2023-09-06T20:39:41,573 INFO [RS:0;jenkins-hbase3:45559 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-09-06T20:39:41,573 INFO [RS:0;jenkins-hbase3:45559 {}] regionserver.HRegionServer(3528): Received CLOSE for 1588230740 2023-09-06T20:39:41,578 INFO [RS:0;jenkins-hbase3:45559 {}] regionserver.HRegionServer(1579): Waiting on 6 regions to close 2023-09-06T20:39:41,578 DEBUG [RS:0;jenkins-hbase3:45559 {}] regionserver.HRegionServer(1583): Online Regions={1588230740=hbase:meta,,1.1588230740, 42b03c0f5b2d2278a9b485cc44396dc1=hbase:namespace,,1694032719521.42b03c0f5b2d2278a9b485cc44396dc1., 87e84cabad1c9957eead562b206d922d=ns2:test-16940327251531,,1694032727923.87e84cabad1c9957eead562b206d922d., 7499612530301bea74e2d6f06f601777=test-1694032725153,,1694032725664.7499612530301bea74e2d6f06f601777., 17f2e8bf9b7a35cfadd958888b7227bd=ns4:test-16940327251533,,1694032731255.17f2e8bf9b7a35cfadd958888b7227bd., de261a559c0224b9046b4ead35ab8e78=backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.} 2023-09-06T20:39:41,579 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-09-06T20:39:41,579 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/hbase/namespace/42b03c0f5b2d2278a9b485cc44396dc1/.tmp/info/a6e03d17f7d04948a154c2756822ea93 2023-09-06T20:39:41,579 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-09-06T20:39:41,579 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-09-06T20:39:41,579 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-09-06T20:39:41,579 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-09-06T20:39:41,579 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=19.18 KB heapSize=33.47 KB 2023-09-06T20:39:41,580 DEBUG [RS:0;jenkins-hbase3:45559 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 17f2e8bf9b7a35cfadd958888b7227bd, 42b03c0f5b2d2278a9b485cc44396dc1, 7499612530301bea74e2d6f06f601777, 87e84cabad1c9957eead562b206d922d, de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:41,587 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/hbase/namespace/42b03c0f5b2d2278a9b485cc44396dc1/.tmp/info/a6e03d17f7d04948a154c2756822ea93 as hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/hbase/namespace/42b03c0f5b2d2278a9b485cc44396dc1/info/a6e03d17f7d04948a154c2756822ea93 2023-09-06T20:39:41,594 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/hbase/namespace/42b03c0f5b2d2278a9b485cc44396dc1/info/a6e03d17f7d04948a154c2756822ea93, entries=7, sequenceid=11, filesize=5.1 K 2023-09-06T20:39:41,595 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 42b03c0f5b2d2278a9b485cc44396dc1 in 46ms, sequenceid=11, compaction requested=false 2023-09-06T20:39:41,610 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/hbase/meta/1588230740/.tmp/info/2c0edf20d2b64e248316d789dcade927 is 159, key is ns2:test-16940327251531,,1694032727923.87e84cabad1c9957eead562b206d922d./info:regioninfo/1694032728709/Put/seqid=0 2023-09-06T20:39:41,611 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/hbase/namespace/42b03c0f5b2d2278a9b485cc44396dc1/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-09-06T20:39:41,612 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:39:41,615 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1694032719521.42b03c0f5b2d2278a9b485cc44396dc1. 2023-09-06T20:39:41,615 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 42b03c0f5b2d2278a9b485cc44396dc1: 2023-09-06T20:39:41,615 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1694032719521.42b03c0f5b2d2278a9b485cc44396dc1. 2023-09-06T20:39:41,615 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 87e84cabad1c9957eead562b206d922d, disabling compactions & flushes 2023-09-06T20:39:41,615 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-16940327251531,,1694032727923.87e84cabad1c9957eead562b206d922d. 2023-09-06T20:39:41,615 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16940327251531,,1694032727923.87e84cabad1c9957eead562b206d922d. 2023-09-06T20:39:41,615 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16940327251531,,1694032727923.87e84cabad1c9957eead562b206d922d. after waiting 0 ms 2023-09-06T20:39:41,615 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16940327251531,,1694032727923.87e84cabad1c9957eead562b206d922d. 2023-09-06T20:39:41,624 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/ns2/test-16940327251531/87e84cabad1c9957eead562b206d922d/recovered.edits/107.seqid, newMaxSeqId=107, maxSeqId=1 2023-09-06T20:39:41,625 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:39:41,626 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-16940327251531,,1694032727923.87e84cabad1c9957eead562b206d922d. 2023-09-06T20:39:41,626 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 87e84cabad1c9957eead562b206d922d: 2023-09-06T20:39:41,626 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-16940327251531,,1694032727923.87e84cabad1c9957eead562b206d922d. 2023-09-06T20:39:41,626 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 7499612530301bea74e2d6f06f601777, disabling compactions & flushes 2023-09-06T20:39:41,626 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1694032725153,,1694032725664.7499612530301bea74e2d6f06f601777. 2023-09-06T20:39:41,626 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1694032725153,,1694032725664.7499612530301bea74e2d6f06f601777. 2023-09-06T20:39:41,626 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1694032725153,,1694032725664.7499612530301bea74e2d6f06f601777. after waiting 0 ms 2023-09-06T20:39:41,626 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1694032725153,,1694032725664.7499612530301bea74e2d6f06f601777. 2023-09-06T20:39:41,629 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/default/test-1694032725153/7499612530301bea74e2d6f06f601777/recovered.edits/107.seqid, newMaxSeqId=107, maxSeqId=1 2023-09-06T20:39:41,630 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:39:41,631 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1694032725153,,1694032725664.7499612530301bea74e2d6f06f601777. 2023-09-06T20:39:41,631 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 7499612530301bea74e2d6f06f601777: 2023-09-06T20:39:41,631 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1694032725153,,1694032725664.7499612530301bea74e2d6f06f601777. 2023-09-06T20:39:41,631 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 17f2e8bf9b7a35cfadd958888b7227bd, disabling compactions & flushes 2023-09-06T20:39:41,632 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-16940327251533,,1694032731255.17f2e8bf9b7a35cfadd958888b7227bd. 2023-09-06T20:39:41,632 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16940327251533,,1694032731255.17f2e8bf9b7a35cfadd958888b7227bd. 2023-09-06T20:39:41,632 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16940327251533,,1694032731255.17f2e8bf9b7a35cfadd958888b7227bd. after waiting 0 ms 2023-09-06T20:39:41,632 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16940327251533,,1694032731255.17f2e8bf9b7a35cfadd958888b7227bd. 2023-09-06T20:39:41,634 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/ns4/test-16940327251533/17f2e8bf9b7a35cfadd958888b7227bd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-09-06T20:39:41,635 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:39:41,635 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-16940327251533,,1694032731255.17f2e8bf9b7a35cfadd958888b7227bd. 2023-09-06T20:39:41,636 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 17f2e8bf9b7a35cfadd958888b7227bd: 2023-09-06T20:39:41,636 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-16940327251533,,1694032731255.17f2e8bf9b7a35cfadd958888b7227bd. 2023-09-06T20:39:41,636 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing de261a559c0224b9046b4ead35ab8e78, disabling compactions & flushes 2023-09-06T20:39:41,636 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:41,636 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:41,636 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. after waiting 0 ms 2023-09-06T20:39:41,636 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:41,636 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing de261a559c0224b9046b4ead35ab8e78 2/2 column families, dataSize=395 B heapSize=1.03 KB 2023-09-06T20:39:41,639 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/a1270fdbd1e74223a363d57a4315cf65 is 355, key is session:backup_1694032764728/session:context/1694032781390/Put/seqid=0 2023-09-06T20:39:41,780 DEBUG [RS:0;jenkins-hbase3:45559 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:41,981 DEBUG [RS:0;jenkins-hbase3:45559 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, de261a559c0224b9046b4ead35ab8e78 2023-09-06T20:39:42,023 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.61 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/hbase/meta/1588230740/.tmp/info/2c0edf20d2b64e248316d789dcade927 2023-09-06T20:39:42,043 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=395 B at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/a1270fdbd1e74223a363d57a4315cf65 2023-09-06T20:39:42,044 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/hbase/meta/1588230740/.tmp/table/0a19233d1a604c24805ab93ec88e8312 is 59, key is ns2:test-16940327251531/table:state/1694032728718/Put/seqid=0 2023-09-06T20:39:42,053 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.58 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/hbase/meta/1588230740/.tmp/table/0a19233d1a604c24805ab93ec88e8312 2023-09-06T20:39:42,054 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/.tmp/session/a1270fdbd1e74223a363d57a4315cf65 as hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/a1270fdbd1e74223a363d57a4315cf65 2023-09-06T20:39:42,060 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/a1270fdbd1e74223a363d57a4315cf65, entries=2, sequenceid=81, filesize=5.3 K 2023-09-06T20:39:42,060 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/hbase/meta/1588230740/.tmp/info/2c0edf20d2b64e248316d789dcade927 as hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/hbase/meta/1588230740/info/2c0edf20d2b64e248316d789dcade927 2023-09-06T20:39:42,061 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~395 B/395, heapSize ~784 B/784, currentSize=0 B/0 for de261a559c0224b9046b4ead35ab8e78 in 425ms, sequenceid=81, compaction requested=false 2023-09-06T20:39:42,067 DEBUG [StoreCloser-backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/21801e1055db41adac61df285d558318, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/9585b49cb83643a185229f157af65936, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/98a5e9c6fde5479191b554626ed8f3c4] to archive 2023-09-06T20:39:42,068 DEBUG [StoreCloser-backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-09-06T20:39:42,071 DEBUG [StoreCloser-backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/21801e1055db41adac61df285d558318 to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/archive/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/21801e1055db41adac61df285d558318 2023-09-06T20:39:42,073 DEBUG [StoreCloser-backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/9585b49cb83643a185229f157af65936 to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/archive/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/9585b49cb83643a185229f157af65936 2023-09-06T20:39:42,073 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/hbase/meta/1588230740/info/2c0edf20d2b64e248316d789dcade927, entries=75, sequenceid=89, filesize=14.5 K 2023-09-06T20:39:42,074 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/hbase/meta/1588230740/.tmp/table/0a19233d1a604c24805ab93ec88e8312 as hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/hbase/meta/1588230740/table/0a19233d1a604c24805ab93ec88e8312 2023-09-06T20:39:42,075 DEBUG [StoreCloser-backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/98a5e9c6fde5479191b554626ed8f3c4 to hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/archive/data/backup/system/de261a559c0224b9046b4ead35ab8e78/session/98a5e9c6fde5479191b554626ed8f3c4 2023-09-06T20:39:42,083 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/backup/system/de261a559c0224b9046b4ead35ab8e78/recovered.edits/84.seqid, newMaxSeqId=84, maxSeqId=75 2023-09-06T20:39:42,083 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:39:42,083 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/hbase/meta/1588230740/table/0a19233d1a604c24805ab93ec88e8312, entries=15, sequenceid=89, filesize=5.8 K 2023-09-06T20:39:42,084 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:42,084 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for de261a559c0224b9046b4ead35ab8e78: 2023-09-06T20:39:42,084 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1694032733559.de261a559c0224b9046b4ead35ab8e78. 2023-09-06T20:39:42,084 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~19.18 KB/19644, heapSize ~33.19 KB/33984, currentSize=0 B/0 for 1588230740 in 505ms, sequenceid=89, compaction requested=false 2023-09-06T20:39:42,100 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/data/hbase/meta/1588230740/recovered.edits/92.seqid, newMaxSeqId=92, maxSeqId=1 2023-09-06T20:39:42,100 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:39:42,100 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-09-06T20:39:42,101 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-09-06T20:39:42,101 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-09-06T20:39:42,101 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-09-06T20:39:42,181 INFO [RS:0;jenkins-hbase3:45559 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,45559,1694032716172; all regions closed. 2023-09-06T20:39:42,191 DEBUG [RS:0;jenkins-hbase3:45559 {}] wal.AbstractFSWAL(1062): Moved 4 WAL file(s) to /user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/oldWALs 2023-09-06T20:39:42,191 INFO [RS:0;jenkins-hbase3:45559 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C45559%2C1694032716172.meta:.meta(num 1694032766884) 2023-09-06T20:39:42,197 DEBUG [RS:0;jenkins-hbase3:45559 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/oldWALs 2023-09-06T20:39:42,197 INFO [RS:0;jenkins-hbase3:45559 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C45559%2C1694032716172:(num 1694032766901) 2023-09-06T20:39:42,197 DEBUG [RS:0;jenkins-hbase3:45559 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:39:42,197 INFO [RS:0;jenkins-hbase3:45559 {}] regionserver.LeaseManager(133): Closed leases 2023-09-06T20:39:42,197 INFO [RS:0;jenkins-hbase3:45559 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2023-09-06T20:39:42,197 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-09-06T20:39:42,199 INFO [RS:0;jenkins-hbase3:45559 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:45559 2023-09-06T20:39:42,207 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,45559,1694032716172] 2023-09-06T20:39:42,208 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,45559,1694032716172; numProcessing=1 2023-09-06T20:39:42,210 INFO [RegionServerTracker-0 {}] master.ServerManager(569): Cluster shutdown set; jenkins-hbase3.apache.org,45559,1694032716172 expired; onlineServers=0 2023-09-06T20:39:42,210 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,41705,1694032713168' ***** 2023-09-06T20:39:42,210 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2531): STOPPED: Cluster shutdown set; onlineServer=0 2023-09-06T20:39:42,211 DEBUG [M:0;jenkins-hbase3:41705 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a1bad97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-09-06T20:39:42,211 INFO [M:0;jenkins-hbase3:41705 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-09-06T20:39:42,254 INFO [M:0;jenkins-hbase3:41705 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@53dd1486{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-09-06T20:39:42,255 INFO [M:0;jenkins-hbase3:41705 {}] server.AbstractConnector(383): Stopped ServerConnector@19e02c9c{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-09-06T20:39:42,255 INFO [M:0;jenkins-hbase3:41705 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-09-06T20:39:42,256 INFO [M:0;jenkins-hbase3:41705 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@5bca7cba{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-09-06T20:39:42,256 INFO [M:0;jenkins-hbase3:41705 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@e213d8e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/hadoop.log.dir/,STOPPED} 2023-09-06T20:39:42,256 INFO [M:0;jenkins-hbase3:41705 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,41705,1694032713168 2023-09-06T20:39:42,257 INFO [M:0;jenkins-hbase3:41705 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,41705,1694032713168; all regions closed. 2023-09-06T20:39:42,257 DEBUG [M:0;jenkins-hbase3:41705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:39:42,257 INFO [M:0;jenkins-hbase3:41705 {}] master.HMaster(1665): Stopping master jetty server 2023-09-06T20:39:42,257 INFO [M:0;jenkins-hbase3:41705 {}] server.AbstractConnector(383): Stopped ServerConnector@4f016b3e{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-09-06T20:39:42,258 DEBUG [M:0;jenkins-hbase3:41705 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-09-06T20:39:42,258 INFO [M:0;jenkins-hbase3:41705 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-09-06T20:39:42,258 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-09-06T20:39:42,258 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1694032718276 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1694032718276,5,FailOnTimeoutGroup] 2023-09-06T20:39:42,258 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1694032718278 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1694032718278,5,FailOnTimeoutGroup] 2023-09-06T20:39:42,258 DEBUG [M:0;jenkins-hbase3:41705 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-09-06T20:39:42,258 DEBUG [M:0;jenkins-hbase3:41705 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-09-06T20:39:42,258 INFO [M:0;jenkins-hbase3:41705 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-09-06T20:39:42,259 DEBUG [M:0;jenkins-hbase3:41705 {}] master.HMaster(1688): Stopping service threads 2023-09-06T20:39:42,259 INFO [M:0;jenkins-hbase3:41705 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-09-06T20:39:42,259 ERROR [M:0;jenkins-hbase3:41705 {}] procedure2.ProcedureExecutor(654): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] 2023-09-06T20:39:42,259 INFO [M:0;jenkins-hbase3:41705 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-09-06T20:39:42,260 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-09-06T20:39:42,260 WARN [M:0;jenkins-hbase3:41705 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-09-06T20:39:42,260 INFO [M:0;jenkins-hbase3:41705 {}] assignment.AssignmentManager(382): Stopping assignment manager 2023-09-06T20:39:42,260 INFO [M:0;jenkins-hbase3:41705 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-09-06T20:39:42,261 DEBUG [M:0;jenkins-hbase3:41705 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-09-06T20:39:42,272 INFO [M:0;jenkins-hbase3:41705 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:39:42,273 DEBUG [M:0;jenkins-hbase3:41705 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:39:42,273 DEBUG [M:0;jenkins-hbase3:41705 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-09-06T20:39:42,273 DEBUG [M:0;jenkins-hbase3:41705 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:39:42,273 INFO [M:0;jenkins-hbase3:41705 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=336.61 KB heapSize=405.63 KB 2023-09-06T20:39:42,285 DEBUG [M:0;jenkins-hbase3:41705 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/70541de16e054868ad564aab81553277 is 88, key is hbase:meta,,1/info:sn/1694032718879/Put/seqid=0 2023-09-06T20:39:42,309 INFO [RS:0;jenkins-hbase3:45559 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,45559,1694032716172; zookeeper connection closed. 2023-09-06T20:39:42,309 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@77e10b5f {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@77e10b5f 2023-09-06T20:39:42,309 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-09-06T20:39:42,690 INFO [M:0;jenkins-hbase3:41705 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=826 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/70541de16e054868ad564aab81553277 2023-09-06T20:39:42,709 DEBUG [M:0;jenkins-hbase3:41705 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/28370cf8d72c48f9809886db21b24bc1 is 1038, key is \x00\x00\x00\x00\x00\x00\x00&/proc:d/1694032741800/Put/seqid=0 2023-09-06T20:39:43,114 INFO [M:0;jenkins-hbase3:41705 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=336.01 KB at sequenceid=826 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/28370cf8d72c48f9809886db21b24bc1 2023-09-06T20:39:43,119 INFO [M:0;jenkins-hbase3:41705 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 28370cf8d72c48f9809886db21b24bc1 2023-09-06T20:39:43,132 DEBUG [M:0;jenkins-hbase3:41705 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/256c8cf07a1f4b3eb63a9e6e5ebe7a0a is 82, key is jenkins-hbase3.apache.org,45559,1694032716172/rs:state/1694032718341/Put/seqid=0 2023-09-06T20:39:43,494 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-09-06T20:39:43,536 INFO [M:0;jenkins-hbase3:41705 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=826 (bloomFilter=true), to=hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/256c8cf07a1f4b3eb63a9e6e5ebe7a0a 2023-09-06T20:39:43,542 DEBUG [M:0;jenkins-hbase3:41705 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/70541de16e054868ad564aab81553277 as hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/70541de16e054868ad564aab81553277 2023-09-06T20:39:43,548 INFO [M:0;jenkins-hbase3:41705 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/70541de16e054868ad564aab81553277, entries=8, sequenceid=826, filesize=5.5 K 2023-09-06T20:39:43,549 DEBUG [M:0;jenkins-hbase3:41705 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/28370cf8d72c48f9809886db21b24bc1 as hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/28370cf8d72c48f9809886db21b24bc1 2023-09-06T20:39:43,554 INFO [M:0;jenkins-hbase3:41705 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 28370cf8d72c48f9809886db21b24bc1 2023-09-06T20:39:43,554 INFO [M:0;jenkins-hbase3:41705 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/28370cf8d72c48f9809886db21b24bc1, entries=89, sequenceid=826, filesize=26.7 K 2023-09-06T20:39:43,555 DEBUG [M:0;jenkins-hbase3:41705 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/256c8cf07a1f4b3eb63a9e6e5ebe7a0a as hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/256c8cf07a1f4b3eb63a9e6e5ebe7a0a 2023-09-06T20:39:43,560 INFO [M:0;jenkins-hbase3:41705 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34451/user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/256c8cf07a1f4b3eb63a9e6e5ebe7a0a, entries=1, sequenceid=826, filesize=5.1 K 2023-09-06T20:39:43,561 INFO [M:0;jenkins-hbase3:41705 {}] regionserver.HRegion(3022): Finished flush of dataSize ~336.61 KB/344686, heapSize ~405.33 KB/415056, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1288ms, sequenceid=826, compaction requested=false 2023-09-06T20:39:43,570 INFO [M:0;jenkins-hbase3:41705 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-06T20:39:43,570 DEBUG [M:0;jenkins-hbase3:41705 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-09-06T20:39:43,574 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/d756d437-0fe2-79de-474a-78519ba592f0/MasterData/WALs/jenkins-hbase3.apache.org,41705,1694032713168/jenkins-hbase3.apache.org%2C41705%2C1694032713168.1694032717455 not finished, retry = 0 2023-09-06T20:39:43,675 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-09-06T20:39:43,675 INFO [M:0;jenkins-hbase3:41705 {}] flush.MasterFlushTableProcedureManager(85): stop: server shutting down. 2023-09-06T20:39:43,675 INFO [M:0;jenkins-hbase3:41705 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-09-06T20:39:43,677 INFO [M:0;jenkins-hbase3:41705 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:41705 2023-09-06T20:39:43,782 INFO [M:0;jenkins-hbase3:41705 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,41705,1694032713168; zookeeper connection closed. 2023-09-06T20:39:43,784 WARN [Listener at jenkins-hbase3.apache.org/41977 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-09-06T20:39:43,789 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-09-06T20:39:43,896 WARN [BP-91659908-172.31.12.81-1694032709610 heartbeating to localhost/127.0.0.1:34451 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-09-06T20:39:43,898 WARN [BP-91659908-172.31.12.81-1694032709610 heartbeating to localhost/127.0.0.1:34451 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-91659908-172.31.12.81-1694032709610 (Datanode Uuid 8df01a43-0f69-4a0f-85c1-859f5e9f0945) service to localhost/127.0.0.1:34451 2023-09-06T20:39:43,900 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/cluster_b47b59a8-e7b8-2208-bc7e-15cb99b544fd/dfs/data/data1/current/BP-91659908-172.31.12.81-1694032709610 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-09-06T20:39:43,901 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a8f663e6-2dc6-d297-f3ff-d2c37833d9d2/cluster_b47b59a8-e7b8-2208-bc7e-15cb99b544fd/dfs/data/data2/current/BP-91659908-172.31.12.81-1694032709610 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-09-06T20:39:43,939 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-09-06T20:39:44,097 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-09-06T20:39:44,097 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-09-06T20:39:44,107 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-09-06T20:39:44,224 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-09-06T20:39:44,270 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16940327251531 2023-09-06T20:39:44,270 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-09-06T20:39:44,270 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-09-06T20:39:44,270 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-09-06T20:39:44,270 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-09-06T20:39:44,270 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-09-06T20:39:44,270 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-09-06T20:39:44,270 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16940327251532 2023-09-06T20:39:44,270 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1694032725153 2023-09-06T20:39:44,270 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16940327251533 2023-09-06T20:39:44,270 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-09-06T20:39:44,333 ERROR [Thread[Thread-275,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-09-06T20:39:44,333 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-09-06T20:39:44,439 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-09-06T20:39:44,443 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-09-06T20:39:44,445 ERROR [Thread[Thread-285,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-09-06T20:39:44,454 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-09-06T20:39:44,556 ERROR [Thread[Thread-250,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-09-06T20:39:44,556 INFO [Listener at jenkins-hbase3.apache.org/41977 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped