2023-07-22 12:25:41,298 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@776aec5c 2023-07-22 12:25:41,313 main DEBUG Took 0.011939 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-07-22 12:25:41,313 main DEBUG PluginManager 'Core' found 129 plugins 2023-07-22 12:25:41,313 main DEBUG PluginManager 'Level' found 0 plugins 2023-07-22 12:25:41,314 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-07-22 12:25:41,315 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:25:41,323 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-07-22 12:25:41,335 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:25:41,336 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:25:41,337 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:25:41,337 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:25:41,338 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:25:41,338 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:25:41,338 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:25:41,339 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:25:41,339 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:25:41,339 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:25:41,340 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:25:41,340 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:25:41,341 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:25:41,341 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:25:41,341 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:25:41,341 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:25:41,342 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:25:41,342 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:25:41,342 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:25:41,343 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:25:41,343 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:25:41,343 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:25:41,344 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:25:41,344 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:25:41,344 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:25:41,344 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-07-22 12:25:41,346 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:25:41,348 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-07-22 12:25:41,350 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-07-22 12:25:41,350 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-07-22 12:25:41,351 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-07-22 12:25:41,352 main DEBUG PluginManager 'Converter' found 47 plugins 2023-07-22 12:25:41,364 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-07-22 12:25:41,366 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-07-22 12:25:41,367 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-07-22 12:25:41,368 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-07-22 12:25:41,368 main DEBUG createAppenders(={Console}) 2023-07-22 12:25:41,369 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@776aec5c initialized 2023-07-22 12:25:41,369 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@776aec5c 2023-07-22 12:25:41,369 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@776aec5c OK. 2023-07-22 12:25:41,370 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-07-22 12:25:41,370 main DEBUG OutputStream closed 2023-07-22 12:25:41,370 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-07-22 12:25:41,370 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-07-22 12:25:41,371 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@3daa422a OK 2023-07-22 12:25:41,413 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-07-22 12:25:41,415 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-07-22 12:25:41,416 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-07-22 12:25:41,417 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-07-22 12:25:41,418 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-07-22 12:25:41,418 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-07-22 12:25:41,418 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-07-22 12:25:41,419 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-07-22 12:25:41,419 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-07-22 12:25:41,419 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-07-22 12:25:41,419 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-07-22 12:25:41,420 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-07-22 12:25:41,420 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-07-22 12:25:41,420 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-07-22 12:25:41,420 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-07-22 12:25:41,421 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-07-22 12:25:41,421 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-07-22 12:25:41,422 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-07-22 12:25:41,425 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-07-22 12:25:41,425 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@15bbf42f) with optional ClassLoader: null 2023-07-22 12:25:41,426 main DEBUG Shutdown hook enabled. Registering a new one. 2023-07-22 12:25:41,426 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@15bbf42f] started OK. 2023-07-22T12:25:41,456 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures timeout: 13 mins 2023-07-22 12:25:41,460 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-07-22 12:25:41,460 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-07-22T12:25:41,776 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c 2023-07-22T12:25:41,781 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-07-22T12:25:41,783 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:25:41,784 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-07-22T12:25:41,788 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/cluster_5c7a65c4-cab7-9c85-3939-092136c4beb7, deleteOnExit=true 2023-07-22T12:25:41,788 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-07-22T12:25:41,788 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/test.cache.data in system properties and HBase conf 2023-07-22T12:25:41,789 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/hadoop.tmp.dir in system properties and HBase conf 2023-07-22T12:25:41,789 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/hadoop.log.dir in system properties and HBase conf 2023-07-22T12:25:41,790 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/mapreduce.cluster.local.dir in system properties and HBase conf 2023-07-22T12:25:41,790 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-07-22T12:25:41,790 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-07-22T12:25:41,901 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-07-22T12:25:42,381 DEBUG [Time-limited test {}] fs.HFileSystem(308): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-07-22T12:25:42,386 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-07-22T12:25:42,386 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-07-22T12:25:42,386 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-07-22T12:25:42,387 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-07-22T12:25:42,387 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-07-22T12:25:42,387 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-07-22T12:25:42,388 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-07-22T12:25:42,388 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/dfs.journalnode.edits.dir in system properties and HBase conf 2023-07-22T12:25:42,388 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-07-22T12:25:42,389 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/nfs.dump.dir in system properties and HBase conf 2023-07-22T12:25:42,389 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/java.io.tmpdir in system properties and HBase conf 2023-07-22T12:25:42,389 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/dfs.journalnode.edits.dir in system properties and HBase conf 2023-07-22T12:25:42,389 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-07-22T12:25:42,390 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-07-22T12:25:42,843 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-07-22T12:25:42,848 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-07-22T12:25:43,133 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-07-22T12:25:43,320 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@3f08d1f2 via org.mortbay.log.Slf4jLog 2023-07-22T12:25:43,336 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:25:43,374 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:25:43,418 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/java.io.tmpdir/Jetty_localhost_43175_hdfs____.8wo5ly/webapp 2023-07-22T12:25:43,563 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:43175 2023-07-22T12:25:43,586 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-07-22T12:25:43,587 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-07-22T12:25:44,133 WARN [Listener at localhost/35695 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-07-22T12:25:44,200 WARN [Listener at localhost/35695 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-07-22T12:25:44,218 WARN [Listener at localhost/35695 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:25:44,223 INFO [Listener at localhost/35695 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:25:44,229 INFO [Listener at localhost/35695 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/java.io.tmpdir/Jetty_localhost_37995_datanode____.ehwcx4/webapp 2023-07-22T12:25:44,337 INFO [Listener at localhost/35695 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:37995 2023-07-22T12:25:44,616 WARN [Listener at localhost/44609 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-07-22T12:25:44,956 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-07-22T12:25:45,135 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x64a088388944e07b: Processing first storage report for DS-940ef494-c177-4702-ad53-b2959f1d3b03 from datanode de9f297b-911d-450f-ae2b-e21372ec3fcc 2023-07-22T12:25:45,137 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x64a088388944e07b: from storage DS-940ef494-c177-4702-ad53-b2959f1d3b03 node DatanodeRegistration(127.0.0.1:35109, datanodeUuid=de9f297b-911d-450f-ae2b-e21372ec3fcc, infoPort=34579, infoSecurePort=0, ipcPort=44609, storageInfo=lv=-57;cid=testClusterID;nsid=1020459575;c=1690028742924), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2023-07-22T12:25:45,137 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x64a088388944e07b: Processing first storage report for DS-536414fc-55a6-45d6-be9f-0977ae0d7fbc from datanode de9f297b-911d-450f-ae2b-e21372ec3fcc 2023-07-22T12:25:45,137 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x64a088388944e07b: from storage DS-536414fc-55a6-45d6-be9f-0977ae0d7fbc node DatanodeRegistration(127.0.0.1:35109, datanodeUuid=de9f297b-911d-450f-ae2b-e21372ec3fcc, infoPort=34579, infoSecurePort=0, ipcPort=44609, storageInfo=lv=-57;cid=testClusterID;nsid=1020459575;c=1690028742924), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-07-22T12:25:45,168 DEBUG [Listener at localhost/44609 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c 2023-07-22T12:25:45,247 INFO [Listener at localhost/44609 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/cluster_5c7a65c4-cab7-9c85-3939-092136c4beb7/zookeeper_0, clientPort=60489, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/cluster_5c7a65c4-cab7-9c85-3939-092136c4beb7/zookeeper_0/version-2, dataDirSize=424 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/cluster_5c7a65c4-cab7-9c85-3939-092136c4beb7/zookeeper_0/version-2, dataLogSize=424 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, serverId=0 2023-07-22T12:25:45,264 INFO [Listener at localhost/44609 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=60489 2023-07-22T12:25:45,272 INFO [Listener at localhost/44609 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:25:45,273 INFO [Listener at localhost/44609 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:25:45,920 INFO [Listener at localhost/44609 {}] util.FSUtils(461): Created version file at hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb with version=8 2023-07-22T12:25:45,921 INFO [Listener at localhost/44609 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/hbase-staging 2023-07-22T12:25:45,939 DEBUG [Listener at localhost/44609 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-07-22T12:25:45,939 DEBUG [Listener at localhost/44609 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-07-22T12:25:45,940 DEBUG [Listener at localhost/44609 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-07-22T12:25:45,940 DEBUG [Listener at localhost/44609 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-07-22T12:25:46,155 DEBUG [Listener at localhost/44609 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-07-22T12:25:46,305 INFO [Listener at localhost/44609 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-07-22T12:25:46,939 INFO [Listener at localhost/44609 {}] client.ConnectionUtils(127): master/jenkins-hbase3:0 server-side Connection retries=3 2023-07-22T12:25:46,985 INFO [Listener at localhost/44609 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-07-22T12:25:46,986 INFO [Listener at localhost/44609 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-07-22T12:25:46,986 INFO [Listener at localhost/44609 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-07-22T12:25:46,986 INFO [Listener at localhost/44609 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-07-22T12:25:46,987 INFO [Listener at localhost/44609 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-07-22T12:25:47,135 INFO [Listener at localhost/44609 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-07-22T12:25:47,240 DEBUG [Listener at localhost/44609 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-07-22T12:25:47,246 INFO [Listener at localhost/44609 {}] ipc.NettyRpcServer(177): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-07-22T12:25:47,294 DEBUG [Listener at localhost/44609 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 23035 (auto-detected) 2023-07-22T12:25:47,296 DEBUG [Listener at localhost/44609 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:1c:ff:fe:e2:13:ea (auto-detected) 2023-07-22T12:25:47,333 INFO [Listener at localhost/44609 {}] ipc.NettyRpcServer(144): Bind to /172.31.12.81:34453 2023-07-22T12:25:47,348 INFO [Listener at localhost/44609 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:25:47,350 INFO [Listener at localhost/44609 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:25:47,367 INFO [Listener at localhost/44609 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:34453 connecting to ZooKeeper ensemble=127.0.0.1:60489 2023-07-22T12:25:47,416 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:344530x0, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-07-22T12:25:47,419 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:34453-0x100005982690000 connected 2023-07-22T12:25:47,446 DEBUG [Listener at localhost/44609 {}] zookeeper.ZKUtil(113): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-07-22T12:25:47,449 DEBUG [Listener at localhost/44609 {}] zookeeper.ZKUtil(113): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-07-22T12:25:47,453 DEBUG [Listener at localhost/44609 {}] zookeeper.ZKUtil(113): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-07-22T12:25:47,463 DEBUG [Listener at localhost/44609 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34453 2023-07-22T12:25:47,463 DEBUG [Listener at localhost/44609 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34453 2023-07-22T12:25:47,464 DEBUG [Listener at localhost/44609 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34453 2023-07-22T12:25:47,466 DEBUG [Listener at localhost/44609 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34453 2023-07-22T12:25:47,470 DEBUG [Listener at localhost/44609 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34453 2023-07-22T12:25:47,524 INFO [Listener at localhost/44609 {}] log.Log(170): Logging initialized @6930ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-07-22T12:25:47,646 INFO [Listener at localhost/44609 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-07-22T12:25:47,647 INFO [Listener at localhost/44609 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-07-22T12:25:47,648 INFO [Listener at localhost/44609 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-07-22T12:25:47,650 INFO [Listener at localhost/44609 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-07-22T12:25:47,650 INFO [Listener at localhost/44609 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-07-22T12:25:47,650 INFO [Listener at localhost/44609 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-07-22T12:25:47,654 INFO [Listener at localhost/44609 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-07-22T12:25:47,797 INFO [Listener at localhost/44609 {}] http.HttpServer(1219): Jetty bound to port 38527 2023-07-22T12:25:47,799 INFO [Listener at localhost/44609 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-07-22T12:25:47,829 INFO [Listener at localhost/44609 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:25:47,832 INFO [Listener at localhost/44609 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@7c4608eb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/hadoop.log.dir/,AVAILABLE} 2023-07-22T12:25:47,833 INFO [Listener at localhost/44609 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:25:47,833 INFO [Listener at localhost/44609 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@1ef78025{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-07-22T12:25:47,896 INFO [Listener at localhost/44609 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-07-22T12:25:47,909 INFO [Listener at localhost/44609 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-07-22T12:25:47,909 INFO [Listener at localhost/44609 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-07-22T12:25:47,911 INFO [Listener at localhost/44609 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2023-07-22T12:25:47,918 INFO [Listener at localhost/44609 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:25:49,064 INFO [Listener at localhost/44609 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@4e64b461{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-07-22T12:25:49,075 INFO [Listener at localhost/44609 {}] server.AbstractConnector(333): Started ServerConnector@23b09d37{HTTP/1.1, (http/1.1)}{0.0.0.0:38527} 2023-07-22T12:25:49,075 INFO [Listener at localhost/44609 {}] server.Server(415): Started @8482ms 2023-07-22T12:25:49,079 INFO [Listener at localhost/44609 {}] master.HMaster(484): hbase.rootdir=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb, hbase.cluster.distributed=false 2023-07-22T12:25:49,135 INFO [Listener at localhost/44609 {}] client.ConnectionUtils(127): regionserver/jenkins-hbase3:0 server-side Connection retries=3 2023-07-22T12:25:49,135 INFO [Listener at localhost/44609 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-07-22T12:25:49,136 INFO [Listener at localhost/44609 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-07-22T12:25:49,136 INFO [Listener at localhost/44609 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-07-22T12:25:49,136 INFO [Listener at localhost/44609 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-07-22T12:25:49,136 INFO [Listener at localhost/44609 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-07-22T12:25:49,144 INFO [Listener at localhost/44609 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-07-22T12:25:49,145 INFO [Listener at localhost/44609 {}] ipc.NettyRpcServer(177): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-07-22T12:25:49,147 INFO [Listener at localhost/44609 {}] ipc.NettyRpcServer(144): Bind to /172.31.12.81:45275 2023-07-22T12:25:49,149 INFO [Listener at localhost/44609 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-07-22T12:25:49,156 DEBUG [Listener at localhost/44609 {}] mob.MobFileCache(120): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-07-22T12:25:49,157 INFO [Listener at localhost/44609 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:25:49,160 INFO [Listener at localhost/44609 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:25:49,161 INFO [Listener at localhost/44609 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:45275 connecting to ZooKeeper ensemble=127.0.0.1:60489 2023-07-22T12:25:49,164 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): regionserver:452750x0, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-07-22T12:25:49,165 DEBUG [Listener at localhost/44609 {}] zookeeper.ZKUtil(113): regionserver:452750x0, quorum=127.0.0.1:60489, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-07-22T12:25:49,165 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:45275-0x100005982690001 connected 2023-07-22T12:25:49,166 DEBUG [Listener at localhost/44609 {}] zookeeper.ZKUtil(113): regionserver:45275-0x100005982690001, quorum=127.0.0.1:60489, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-07-22T12:25:49,167 DEBUG [Listener at localhost/44609 {}] zookeeper.ZKUtil(113): regionserver:45275-0x100005982690001, quorum=127.0.0.1:60489, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-07-22T12:25:49,167 DEBUG [Listener at localhost/44609 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45275 2023-07-22T12:25:49,168 DEBUG [Listener at localhost/44609 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45275 2023-07-22T12:25:49,168 DEBUG [Listener at localhost/44609 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45275 2023-07-22T12:25:49,168 DEBUG [Listener at localhost/44609 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45275 2023-07-22T12:25:49,169 DEBUG [Listener at localhost/44609 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45275 2023-07-22T12:25:49,172 INFO [Listener at localhost/44609 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-07-22T12:25:49,172 INFO [Listener at localhost/44609 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-07-22T12:25:49,172 INFO [Listener at localhost/44609 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-07-22T12:25:49,173 INFO [Listener at localhost/44609 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-07-22T12:25:49,173 INFO [Listener at localhost/44609 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-07-22T12:25:49,173 INFO [Listener at localhost/44609 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-07-22T12:25:49,173 INFO [Listener at localhost/44609 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-07-22T12:25:49,175 INFO [Listener at localhost/44609 {}] http.HttpServer(1219): Jetty bound to port 44835 2023-07-22T12:25:49,175 INFO [Listener at localhost/44609 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-07-22T12:25:49,177 INFO [Listener at localhost/44609 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:25:49,177 INFO [Listener at localhost/44609 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@5c2bf6c1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/hadoop.log.dir/,AVAILABLE} 2023-07-22T12:25:49,177 INFO [Listener at localhost/44609 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:25:49,177 INFO [Listener at localhost/44609 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@5fe3a699{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-07-22T12:25:49,188 INFO [Listener at localhost/44609 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-07-22T12:25:49,189 INFO [Listener at localhost/44609 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-07-22T12:25:49,189 INFO [Listener at localhost/44609 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-07-22T12:25:49,190 INFO [Listener at localhost/44609 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-07-22T12:25:49,190 INFO [Listener at localhost/44609 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:25:49,195 INFO [Listener at localhost/44609 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@40aab786{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-07-22T12:25:49,195 INFO [Listener at localhost/44609 {}] server.AbstractConnector(333): Started ServerConnector@135ebf2f{HTTP/1.1, (http/1.1)}{0.0.0.0:44835} 2023-07-22T12:25:49,196 INFO [Listener at localhost/44609 {}] server.Server(415): Started @8602ms 2023-07-22T12:25:49,200 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-07-22T12:25:49,203 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@76516e23{HTTP/1.1, (http/1.1)}{0.0.0.0:46711} 2023-07-22T12:25:49,203 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @8610ms 2023-07-22T12:25:49,203 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2387): Adding backup master ZNode /1/backup-masters/jenkins-hbase3.apache.org,34453,1690028746140 2023-07-22T12:25:49,215 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-07-22T12:25:49,215 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45275-0x100005982690001, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-07-22T12:25:49,217 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase3.apache.org,34453,1690028746140 2023-07-22T12:25:49,224 DEBUG [M:0;jenkins-hbase3:34453 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:34453 2023-07-22T12:25:49,235 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-07-22T12:25:49,235 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45275-0x100005982690001, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-07-22T12:25:49,235 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:25:49,235 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45275-0x100005982690001, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:25:49,236 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Set watcher on existing znode=/1/master 2023-07-22T12:25:49,237 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase3.apache.org,34453,1690028746140 from backup master directory 2023-07-22T12:25:49,237 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Set watcher on existing znode=/1/master 2023-07-22T12:25:49,241 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45275-0x100005982690001, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-07-22T12:25:49,241 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase3.apache.org,34453,1690028746140 2023-07-22T12:25:49,241 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-07-22T12:25:49,242 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-07-22T12:25:49,242 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,34453,1690028746140 2023-07-22T12:25:49,245 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-07-22T12:25:49,247 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-07-22T12:25:49,324 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(610): Created cluster ID file at hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/hbase.id with ID: 9223ddc0-a368-4113-9d5a-cf7e80b497d8 2023-07-22T12:25:49,363 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:25:49,380 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45275-0x100005982690001, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:25:49,380 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:25:49,424 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:25:49,427 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-07-22T12:25:49,450 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(270): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-07-22T12:25:49,450 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(276): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-07-22T12:25:49,451 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(285): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:283) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:310) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:950) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:573) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:570) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-07-22T12:25:49,462 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:950) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:573) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:570) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-07-22T12:25:49,463 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-07-22T12:25:49,941 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/MasterData/data/master/store 2023-07-22T12:25:50,367 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-07-22T12:25:50,367 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:25:50,368 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-07-22T12:25:50,368 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:25:50,368 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:25:50,368 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-07-22T12:25:50,369 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:25:50,369 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:25:50,369 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-07-22T12:25:50,372 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/MasterData/data/master/store/.initializing 2023-07-22T12:25:50,372 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/MasterData/WALs/jenkins-hbase3.apache.org,34453,1690028746140 2023-07-22T12:25:50,382 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-07-22T12:25:50,397 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C34453%2C1690028746140, suffix=, logDir=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/MasterData/WALs/jenkins-hbase3.apache.org,34453,1690028746140, archiveDir=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/MasterData/oldWALs, maxLogs=10 2023-07-22T12:25:50,435 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/MasterData/WALs/jenkins-hbase3.apache.org,34453,1690028746140/jenkins-hbase3.apache.org%2C34453%2C1690028746140.1690028750404, exclude list is [], retry=0 2023-07-22T12:25:50,461 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35109,DS-940ef494-c177-4702-ad53-b2959f1d3b03,DISK] 2023-07-22T12:25:50,469 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:346) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:430) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:424) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:482) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:477) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:489) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-07-22T12:25:50,539 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/MasterData/WALs/jenkins-hbase3.apache.org,34453,1690028746140/jenkins-hbase3.apache.org%2C34453%2C1690028746140.1690028750404 2023-07-22T12:25:50,540 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:35109,DS-940ef494-c177-4702-ad53-b2959f1d3b03,DISK]] 2023-07-22T12:25:50,541 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:25:50,541 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:25:50,545 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:25:50,547 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:25:50,614 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:25:50,649 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-07-22T12:25:50,655 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:25:50,659 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:25:50,659 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:25:50,662 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-07-22T12:25:50,662 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:25:50,663 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:25:50,663 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:25:50,666 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-07-22T12:25:50,666 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:25:50,667 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:25:50,667 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:25:50,670 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-07-22T12:25:50,670 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:25:50,671 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:25:50,675 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:25:50,676 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:25:50,692 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-07-22T12:25:50,695 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:25:50,700 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:25:50,701 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11189813760, jitterRate=0.04213261604309082}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-07-22T12:25:50,708 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-07-22T12:25:50,709 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-07-22T12:25:50,744 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e148272, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:25:50,800 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(859): No meta location available on zookeeper, skip migrating... 2023-07-22T12:25:50,821 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-07-22T12:25:50,821 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(564): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-07-22T12:25:50,824 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-07-22T12:25:50,826 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(584): Recovered RegionProcedureStore lease in 1 msec 2023-07-22T12:25:50,833 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(598): Loaded RegionProcedureStore in 6 msec 2023-07-22T12:25:50,833 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-07-22T12:25:50,869 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-07-22T12:25:50,886 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-07-22T12:25:50,888 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-07-22T12:25:50,892 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(150): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-07-22T12:25:50,893 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-07-22T12:25:50,895 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-07-22T12:25:50,898 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-07-22T12:25:50,904 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-07-22T12:25:50,906 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-07-22T12:25:50,906 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-07-22T12:25:50,908 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-07-22T12:25:50,923 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-07-22T12:25:50,925 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-07-22T12:25:50,932 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-07-22T12:25:50,932 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45275-0x100005982690001, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-07-22T12:25:50,932 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:25:50,932 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45275-0x100005982690001, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:25:50,933 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(803): Active/primary master=jenkins-hbase3.apache.org,34453,1690028746140, sessionid=0x100005982690000, setting cluster-up flag (Was=false) 2023-07-22T12:25:50,943 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-07-22T12:25:50,950 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:25:50,950 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45275-0x100005982690001, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:25:50,956 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-07-22T12:25:50,957 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,34453,1690028746140 2023-07-22T12:25:50,961 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:25:50,961 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45275-0x100005982690001, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:25:50,966 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-07-22T12:25:50,968 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,34453,1690028746140 2023-07-22T12:25:50,974 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:25:50,974 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45275-0x100005982690001, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:25:50,978 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-07-22T12:25:50,979 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,34453,1690028746140 2023-07-22T12:25:51,010 DEBUG [RS:0;jenkins-hbase3:45275 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:45275 2023-07-22T12:25:51,011 INFO [RS:0;jenkins-hbase3:45275 {}] regionserver.HRegionServer(997): ClusterId : 9223ddc0-a368-4113-9d5a-cf7e80b497d8 2023-07-22T12:25:51,014 INFO [RS:0;jenkins-hbase3:45275 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-07-22T12:25:51,018 DEBUG [RS:0;jenkins-hbase3:45275 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-07-22T12:25:51,025 DEBUG [RS:0;jenkins-hbase3:45275 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-07-22T12:25:51,025 DEBUG [RS:0;jenkins-hbase3:45275 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-07-22T12:25:51,029 DEBUG [RS:0;jenkins-hbase3:45275 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-07-22T12:25:51,029 DEBUG [RS:0;jenkins-hbase3:45275 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-07-22T12:25:51,032 DEBUG [RS:0;jenkins-hbase3:45275 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-07-22T12:25:51,033 DEBUG [RS:0;jenkins-hbase3:45275 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d468ac1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:25:51,036 DEBUG [RS:0;jenkins-hbase3:45275 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1573c1f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-07-22T12:25:51,041 INFO [RS:0;jenkins-hbase3:45275 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-07-22T12:25:51,041 INFO [RS:0;jenkins-hbase3:45275 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-07-22T12:25:51,041 DEBUG [RS:0;jenkins-hbase3:45275 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-07-22T12:25:51,045 INFO [RS:0;jenkins-hbase3:45275 {}] regionserver.HRegionServer(3023): reportForDuty to master=jenkins-hbase3.apache.org,34453,1690028746140 with isa=jenkins-hbase3.apache.org/172.31.12.81:45275, startcode=1690028749134 2023-07-22T12:25:51,053 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver loaded, priority=536870911. 2023-07-22T12:25:51,068 DEBUG [RS:0;jenkins-hbase3:45275 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-07-22T12:25:51,153 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:58825, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-07-22T12:25:51,154 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-07-22T12:25:51,163 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34453 {}] ipc.MetricsHBaseServer(144): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3180) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:588) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-07-22T12:25:51,168 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-07-22T12:25:51,175 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-07-22T12:25:51,188 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,34453,1690028746140 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-07-22T12:25:51,193 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-07-22T12:25:51,194 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-07-22T12:25:51,194 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-07-22T12:25:51,194 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-07-22T12:25:51,194 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-07-22T12:25:51,194 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:25:51,194 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-07-22T12:25:51,194 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:25:51,202 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1690028781202 2023-07-22T12:25:51,207 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-07-22T12:25:51,210 DEBUG [RS:0;jenkins-hbase3:45275 {}] regionserver.HRegionServer(3042): Master is not running yet 2023-07-22T12:25:51,210 WARN [RS:0;jenkins-hbase3:45275 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 100 ms and then retrying. 2023-07-22T12:25:51,216 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-07-22T12:25:51,216 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-07-22T12:25:51,217 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-07-22T12:25:51,223 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:25:51,224 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-07-22T12:25:51,228 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-07-22T12:25:51,228 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-07-22T12:25:51,229 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-07-22T12:25:51,229 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-07-22T12:25:51,229 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-07-22T12:25:51,230 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-07-22T12:25:51,232 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-07-22T12:25:51,235 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-07-22T12:25:51,235 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-07-22T12:25:51,248 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x386f7afb to 127.0.0.1:60489 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:25:51,248 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-07-22T12:25:51,248 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb 2023-07-22T12:25:51,271 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d264960, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:25:51,272 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-07-22T12:25:51,273 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:25:51,275 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-07-22T12:25:51,276 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-07-22T12:25:51,276 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-07-22T12:25:51,277 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1690028751277,5,FailOnTimeoutGroup] 2023-07-22T12:25:51,278 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1690028751278,5,FailOnTimeoutGroup] 2023-07-22T12:25:51,278 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-07-22T12:25:51,278 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1634): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-07-22T12:25:51,278 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-07-22T12:25:51,279 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:25:51,279 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:25:51,280 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-07-22T12:25:51,280 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-07-22T12:25:51,280 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-07-22T12:25:51,282 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-07-22T12:25:51,282 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:25:51,283 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:25:51,283 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-07-22T12:25:51,286 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-07-22T12:25:51,286 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:25:51,287 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:25:51,288 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/hbase/meta/1588230740 2023-07-22T12:25:51,289 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/hbase/meta/1588230740 2023-07-22T12:25:51,292 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-07-22T12:25:51,296 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-07-22T12:25:51,300 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:25:51,301 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10309440000, jitterRate=-0.03985857963562012}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-07-22T12:25:51,304 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-07-22T12:25:51,304 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-07-22T12:25:51,304 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-07-22T12:25:51,304 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-07-22T12:25:51,304 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-07-22T12:25:51,304 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-07-22T12:25:51,306 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-07-22T12:25:51,306 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-07-22T12:25:51,309 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-07-22T12:25:51,309 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-07-22T12:25:51,312 INFO [RS:0;jenkins-hbase3:45275 {}] regionserver.HRegionServer(3023): reportForDuty to master=jenkins-hbase3.apache.org,34453,1690028746140 with isa=jenkins-hbase3.apache.org/172.31.12.81:45275, startcode=1690028749134 2023-07-22T12:25:51,317 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34453 {}] master.ServerManager(403): Registering regionserver=jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:25:51,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-07-22T12:25:51,330 DEBUG [RS:0;jenkins-hbase3:45275 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb 2023-07-22T12:25:51,330 DEBUG [RS:0;jenkins-hbase3:45275 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:35695 2023-07-22T12:25:51,330 DEBUG [RS:0;jenkins-hbase3:45275 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=38527 2023-07-22T12:25:51,334 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-07-22T12:25:51,339 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-07-22T12:25:51,339 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-07-22T12:25:51,340 DEBUG [RS:0;jenkins-hbase3:45275 {}] zookeeper.ZKUtil(111): regionserver:45275-0x100005982690001, quorum=127.0.0.1:60489, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:25:51,340 WARN [RS:0;jenkins-hbase3:45275 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-07-22T12:25:51,341 INFO [RS:0;jenkins-hbase3:45275 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-07-22T12:25:51,341 DEBUG [RS:0;jenkins-hbase3:45275 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:25:51,344 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,45275,1690028749134] 2023-07-22T12:25:51,358 DEBUG [RS:0;jenkins-hbase3:45275 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-07-22T12:25:51,368 INFO [RS:0;jenkins-hbase3:45275 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-07-22T12:25:51,381 INFO [RS:0;jenkins-hbase3:45275 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-07-22T12:25:51,385 INFO [RS:0;jenkins-hbase3:45275 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-07-22T12:25:51,386 INFO [RS:0;jenkins-hbase3:45275 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-07-22T12:25:51,386 INFO [RS:0;jenkins-hbase3:45275 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-07-22T12:25:51,396 INFO [RS:0;jenkins-hbase3:45275 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-07-22T12:25:51,396 DEBUG [RS:0;jenkins-hbase3:45275 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:25:51,396 DEBUG [RS:0;jenkins-hbase3:45275 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:25:51,396 DEBUG [RS:0;jenkins-hbase3:45275 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:25:51,396 DEBUG [RS:0;jenkins-hbase3:45275 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:25:51,396 DEBUG [RS:0;jenkins-hbase3:45275 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:25:51,397 DEBUG [RS:0;jenkins-hbase3:45275 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-07-22T12:25:51,397 DEBUG [RS:0;jenkins-hbase3:45275 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:25:51,397 DEBUG [RS:0;jenkins-hbase3:45275 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:25:51,397 DEBUG [RS:0;jenkins-hbase3:45275 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:25:51,397 DEBUG [RS:0;jenkins-hbase3:45275 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:25:51,397 DEBUG [RS:0;jenkins-hbase3:45275 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:25:51,397 DEBUG [RS:0;jenkins-hbase3:45275 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-07-22T12:25:51,398 INFO [RS:0;jenkins-hbase3:45275 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-07-22T12:25:51,398 INFO [RS:0;jenkins-hbase3:45275 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-07-22T12:25:51,398 INFO [RS:0;jenkins-hbase3:45275 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-07-22T12:25:51,398 INFO [RS:0;jenkins-hbase3:45275 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-07-22T12:25:51,399 INFO [RS:0;jenkins-hbase3:45275 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,45275,1690028749134-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-07-22T12:25:51,414 INFO [RS:0;jenkins-hbase3:45275 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-07-22T12:25:51,418 INFO [RS:0;jenkins-hbase3:45275 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,45275,1690028749134-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-07-22T12:25:51,434 INFO [RS:0;jenkins-hbase3:45275 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,45275,1690028749134 started 2023-07-22T12:25:51,434 INFO [RS:0;jenkins-hbase3:45275 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,45275,1690028749134, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:45275, sessionid=0x100005982690001 2023-07-22T12:25:51,434 DEBUG [RS:0;jenkins-hbase3:45275 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-07-22T12:25:51,434 DEBUG [RS:0;jenkins-hbase3:45275 {}] flush.RegionServerFlushTableProcedureManager(106): Start region server flush procedure manager jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:25:51,434 DEBUG [RS:0;jenkins-hbase3:45275 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,45275,1690028749134' 2023-07-22T12:25:51,434 DEBUG [RS:0;jenkins-hbase3:45275 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-07-22T12:25:51,435 DEBUG [RS:0;jenkins-hbase3:45275 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-07-22T12:25:51,436 DEBUG [RS:0;jenkins-hbase3:45275 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-07-22T12:25:51,436 DEBUG [RS:0;jenkins-hbase3:45275 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-07-22T12:25:51,436 DEBUG [RS:0;jenkins-hbase3:45275 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,45275,1690028749134' 2023-07-22T12:25:51,436 DEBUG [RS:0;jenkins-hbase3:45275 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:25:51,437 DEBUG [RS:0;jenkins-hbase3:45275 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:25:51,437 INFO [RS:0;jenkins-hbase3:45275 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-07-22T12:25:51,437 DEBUG [RS:0;jenkins-hbase3:45275 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-07-22T12:25:51,437 DEBUG [RS:0;jenkins-hbase3:45275 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-07-22T12:25:51,437 DEBUG [RS:0;jenkins-hbase3:45275 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:25:51,437 DEBUG [RS:0;jenkins-hbase3:45275 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,45275,1690028749134' 2023-07-22T12:25:51,437 DEBUG [RS:0;jenkins-hbase3:45275 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-07-22T12:25:51,438 DEBUG [RS:0;jenkins-hbase3:45275 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-07-22T12:25:51,438 DEBUG [RS:0;jenkins-hbase3:45275 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-07-22T12:25:51,439 INFO [RS:0;jenkins-hbase3:45275 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-07-22T12:25:51,439 INFO [RS:0;jenkins-hbase3:45275 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-07-22T12:25:51,500 WARN [jenkins-hbase3:34453 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-07-22T12:25:51,549 INFO [RS:0;jenkins-hbase3:45275 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-07-22T12:25:51,552 INFO [RS:0;jenkins-hbase3:45275 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C45275%2C1690028749134, suffix=, logDir=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134, archiveDir=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/oldWALs, maxLogs=32 2023-07-22T12:25:51,564 DEBUG [RS:0;jenkins-hbase3:45275 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.1690028751553, exclude list is [], retry=0 2023-07-22T12:25:51,570 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35109,DS-940ef494-c177-4702-ad53-b2959f1d3b03,DISK] 2023-07-22T12:25:51,574 INFO [RS:0;jenkins-hbase3:45275 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.1690028751553 2023-07-22T12:25:51,575 DEBUG [RS:0;jenkins-hbase3:45275 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:35109,DS-940ef494-c177-4702-ad53-b2959f1d3b03,DISK]] 2023-07-22T12:25:51,752 DEBUG [jenkins-hbase3:34453 {}] assignment.AssignmentManager(2303): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-07-22T12:25:51,760 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:25:51,785 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,45275,1690028749134, state=OPENING 2023-07-22T12:25:51,795 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-07-22T12:25:51,801 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:25:51,801 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45275-0x100005982690001, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:25:51,802 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-07-22T12:25:51,802 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-07-22T12:25:51,805 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,45275,1690028749134}] 2023-07-22T12:25:52,003 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(721): New admin connection to jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:25:52,005 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=AdminService, sasl=false 2023-07-22T12:25:52,009 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:46784, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-07-22T12:25:52,023 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-07-22T12:25:52,024 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-07-22T12:25:52,025 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-07-22T12:25:52,028 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C45275%2C1690028749134.meta, suffix=.meta, logDir=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134, archiveDir=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/oldWALs, maxLogs=32 2023-07-22T12:25:52,041 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.meta.1690028752029.meta, exclude list is [], retry=0 2023-07-22T12:25:52,045 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35109,DS-940ef494-c177-4702-ad53-b2959f1d3b03,DISK] 2023-07-22T12:25:52,050 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.meta.1690028752029.meta 2023-07-22T12:25:52,050 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:35109,DS-940ef494-c177-4702-ad53-b2959f1d3b03,DISK]] 2023-07-22T12:25:52,050 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:25:52,054 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:25:52,055 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-07-22T12:25:52,071 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-07-22T12:25:52,072 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-07-22T12:25:52,076 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-07-22T12:25:52,077 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:25:52,077 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-07-22T12:25:52,077 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-07-22T12:25:52,080 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-07-22T12:25:52,081 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-07-22T12:25:52,081 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:25:52,082 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:25:52,082 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-07-22T12:25:52,083 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-07-22T12:25:52,083 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:25:52,084 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:25:52,084 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-07-22T12:25:52,085 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-07-22T12:25:52,085 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:25:52,086 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:25:52,087 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/hbase/meta/1588230740 2023-07-22T12:25:52,090 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/hbase/meta/1588230740 2023-07-22T12:25:52,092 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-07-22T12:25:52,095 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-07-22T12:25:52,096 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11724829600, jitterRate=0.09195984899997711}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-07-22T12:25:52,097 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-07-22T12:25:52,109 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2552): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1690028751994 2023-07-22T12:25:52,130 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2579): Finished post open deploy task for hbase:meta,,1.1588230740 2023-07-22T12:25:52,131 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:25:52,131 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-07-22T12:25:52,133 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,45275,1690028749134, state=OPEN 2023-07-22T12:25:52,136 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-07-22T12:25:52,136 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45275-0x100005982690001, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-07-22T12:25:52,136 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-07-22T12:25:52,136 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-07-22T12:25:52,140 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=3, resume processing ppid=2 2023-07-22T12:25:52,140 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,45275,1690028749134 in 331 msec 2023-07-22T12:25:52,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=2, resume processing ppid=1 2023-07-22T12:25:52,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 822 msec 2023-07-22T12:25:52,154 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.0920 sec 2023-07-22T12:25:52,154 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1062): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1690028752154, completionTime=-1 2023-07-22T12:25:52,154 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(830): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-07-22T12:25:52,154 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1629): Joining cluster... 2023-07-22T12:25:52,402 DEBUG [hconnection-0x67511e0c-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:25:52,406 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:46790, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:25:52,422 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1641): Number of RegionServers=1 2023-07-22T12:25:52,422 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1690028812422 2023-07-22T12:25:52,422 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1690028872422 2023-07-22T12:25:52,422 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1648): Joined the cluster in 267 msec 2023-07-22T12:25:52,445 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,34453,1690028746140-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-07-22T12:25:52,445 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,34453,1690028746140-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-07-22T12:25:52,445 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,34453,1690028746140-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-07-22T12:25:52,447 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:34453, period=300000, unit=MILLISECONDS is enabled. 2023-07-22T12:25:52,448 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-07-22T12:25:52,455 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-07-22T12:25:52,469 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-07-22T12:25:52,471 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2367): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-07-22T12:25:52,482 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-07-22T12:25:52,486 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:25:52,488 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:25:52,490 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:25:52,516 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 281efe51d339c53c39818444fc3ed1c9, NAME => 'hbase:namespace,,1690028752470.281efe51d339c53c39818444fc3ed1c9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb 2023-07-22T12:25:52,938 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1690028752470.281efe51d339c53c39818444fc3ed1c9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:25:52,938 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing 281efe51d339c53c39818444fc3ed1c9, disabling compactions & flushes 2023-07-22T12:25:52,938 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1690028752470.281efe51d339c53c39818444fc3ed1c9. 2023-07-22T12:25:52,938 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1690028752470.281efe51d339c53c39818444fc3ed1c9. 2023-07-22T12:25:52,938 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1690028752470.281efe51d339c53c39818444fc3ed1c9. after waiting 0 ms 2023-07-22T12:25:52,938 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1690028752470.281efe51d339c53c39818444fc3ed1c9. 2023-07-22T12:25:52,938 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1690028752470.281efe51d339c53c39818444fc3ed1c9. 2023-07-22T12:25:52,938 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for 281efe51d339c53c39818444fc3ed1c9: 2023-07-22T12:25:52,941 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:25:52,956 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1690028752470.281efe51d339c53c39818444fc3ed1c9.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1690028752943"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690028752943"}]},"ts":"1690028752943"} 2023-07-22T12:25:52,995 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:25:52,998 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:25:53,006 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028752998"}]},"ts":"1690028752998"} 2023-07-22T12:25:53,011 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-07-22T12:25:53,016 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=281efe51d339c53c39818444fc3ed1c9, ASSIGN}] 2023-07-22T12:25:53,019 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=281efe51d339c53c39818444fc3ed1c9, ASSIGN 2023-07-22T12:25:53,021 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=281efe51d339c53c39818444fc3ed1c9, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45275,1690028749134; forceNewPlan=false, retain=false 2023-07-22T12:25:53,172 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=281efe51d339c53c39818444fc3ed1c9, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:25:53,179 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 281efe51d339c53c39818444fc3ed1c9, server=jenkins-hbase3.apache.org,45275,1690028749134}] 2023-07-22T12:25:53,338 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1690028752470.281efe51d339c53c39818444fc3ed1c9. 2023-07-22T12:25:53,339 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => 281efe51d339c53c39818444fc3ed1c9, NAME => 'hbase:namespace,,1690028752470.281efe51d339c53c39818444fc3ed1c9.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:25:53,340 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:25:53,340 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 281efe51d339c53c39818444fc3ed1c9 2023-07-22T12:25:53,340 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1690028752470.281efe51d339c53c39818444fc3ed1c9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:25:53,341 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for 281efe51d339c53c39818444fc3ed1c9 2023-07-22T12:25:53,341 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for 281efe51d339c53c39818444fc3ed1c9 2023-07-22T12:25:53,343 INFO [StoreOpener-281efe51d339c53c39818444fc3ed1c9-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 281efe51d339c53c39818444fc3ed1c9 2023-07-22T12:25:53,345 INFO [StoreOpener-281efe51d339c53c39818444fc3ed1c9-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 281efe51d339c53c39818444fc3ed1c9 columnFamilyName info 2023-07-22T12:25:53,345 DEBUG [StoreOpener-281efe51d339c53c39818444fc3ed1c9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:25:53,346 INFO [StoreOpener-281efe51d339c53c39818444fc3ed1c9-1 {}] regionserver.HStore(324): Store=281efe51d339c53c39818444fc3ed1c9/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:25:53,347 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/hbase/namespace/281efe51d339c53c39818444fc3ed1c9 2023-07-22T12:25:53,348 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/hbase/namespace/281efe51d339c53c39818444fc3ed1c9 2023-07-22T12:25:53,352 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for 281efe51d339c53c39818444fc3ed1c9 2023-07-22T12:25:53,355 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/hbase/namespace/281efe51d339c53c39818444fc3ed1c9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:25:53,356 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened 281efe51d339c53c39818444fc3ed1c9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9923654720, jitterRate=-0.07578763365745544}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:25:53,357 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for 281efe51d339c53c39818444fc3ed1c9: 2023-07-22T12:25:53,359 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2552): Post open deploy tasks for hbase:namespace,,1690028752470.281efe51d339c53c39818444fc3ed1c9., pid=6, masterSystemTime=1690028753333 2023-07-22T12:25:53,364 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=281efe51d339c53c39818444fc3ed1c9, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:25:53,365 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2579): Finished post open deploy task for hbase:namespace,,1690028752470.281efe51d339c53c39818444fc3ed1c9. 2023-07-22T12:25:53,366 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1690028752470.281efe51d339c53c39818444fc3ed1c9. 2023-07-22T12:25:53,373 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=6, resume processing ppid=5 2023-07-22T12:25:53,373 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 281efe51d339c53c39818444fc3ed1c9, server=jenkins-hbase3.apache.org,45275,1690028749134 in 190 msec 2023-07-22T12:25:53,377 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=5, resume processing ppid=4 2023-07-22T12:25:53,377 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=281efe51d339c53c39818444fc3ed1c9, ASSIGN in 357 msec 2023-07-22T12:25:53,379 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:25:53,379 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028753379"}]},"ts":"1690028753379"} 2023-07-22T12:25:53,382 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-07-22T12:25:53,385 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:25:53,386 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-07-22T12:25:53,388 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 913 msec 2023-07-22T12:25:53,388 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-07-22T12:25:53,389 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45275-0x100005982690001, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:25:53,389 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:25:53,438 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-07-22T12:25:53,453 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:25:53,459 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 27 msec 2023-07-22T12:25:53,463 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-07-22T12:25:53,473 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:25:53,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 15 msec 2023-07-22T12:25:53,490 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-07-22T12:25:53,493 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-07-22T12:25:53,493 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1192): Master has completed initialization 4.251sec 2023-07-22T12:25:53,495 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-07-22T12:25:53,496 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-07-22T12:25:53,498 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-07-22T12:25:53,498 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-07-22T12:25:53,499 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-07-22T12:25:53,500 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,34453,1690028746140-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-07-22T12:25:53,501 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,34453,1690028746140-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-07-22T12:25:53,514 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1295): Balancer post startup initialization complete, took 0 seconds 2023-07-22T12:25:53,515 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-07-22T12:25:53,525 DEBUG [Listener at localhost/44609 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x74ad45e0 to 127.0.0.1:60489 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:25:53,529 DEBUG [Listener at localhost/44609 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c891eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:25:53,534 DEBUG [Listener at localhost/44609 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-07-22T12:25:53,534 DEBUG [Listener at localhost/44609 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-07-22T12:25:53,547 DEBUG [hconnection-0x7d45fd63-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:25:53,557 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:46792, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:25:53,567 INFO [Listener at localhost/44609 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,34453,1690028746140 2023-07-22T12:25:53,567 INFO [Listener at localhost/44609 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-07-22T12:25:53,567 INFO [Listener at localhost/44609 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/test.cache.data in system properties and HBase conf 2023-07-22T12:25:53,567 INFO [Listener at localhost/44609 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/hadoop.tmp.dir in system properties and HBase conf 2023-07-22T12:25:53,567 INFO [Listener at localhost/44609 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/hadoop.log.dir in system properties and HBase conf 2023-07-22T12:25:53,568 INFO [Listener at localhost/44609 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/mapreduce.cluster.local.dir in system properties and HBase conf 2023-07-22T12:25:53,568 INFO [Listener at localhost/44609 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-07-22T12:25:53,568 INFO [Listener at localhost/44609 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-07-22T12:25:53,568 INFO [Listener at localhost/44609 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-07-22T12:25:53,568 INFO [Listener at localhost/44609 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-07-22T12:25:53,568 INFO [Listener at localhost/44609 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-07-22T12:25:53,568 INFO [Listener at localhost/44609 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-07-22T12:25:53,568 INFO [Listener at localhost/44609 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-07-22T12:25:53,568 INFO [Listener at localhost/44609 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-07-22T12:25:53,568 INFO [Listener at localhost/44609 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-07-22T12:25:53,568 INFO [Listener at localhost/44609 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/dfs.journalnode.edits.dir in system properties and HBase conf 2023-07-22T12:25:53,568 INFO [Listener at localhost/44609 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-07-22T12:25:53,568 INFO [Listener at localhost/44609 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/nfs.dump.dir in system properties and HBase conf 2023-07-22T12:25:53,568 INFO [Listener at localhost/44609 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/java.io.tmpdir in system properties and HBase conf 2023-07-22T12:25:53,569 INFO [Listener at localhost/44609 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/dfs.journalnode.edits.dir in system properties and HBase conf 2023-07-22T12:25:53,569 INFO [Listener at localhost/44609 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-07-22T12:25:53,569 INFO [Listener at localhost/44609 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-07-22T12:25:55,425 WARN [Thread-249 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:25:55,771 INFO [Thread-249 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:25:55,776 INFO [Thread-249 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_38703_jobhistory____.cxcf41/webapp 2023-07-22T12:25:55,942 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-07-22T12:25:55,942 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-07-22T12:25:55,942 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-07-22T12:25:55,946 INFO [Thread-249 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-07-22T12:25:56,032 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:25:56,149 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:25:56,301 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-07-22T12:25:56,302 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-07-22T12:25:56,311 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-07-22T12:25:56,312 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-07-22T12:25:56,314 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: Master,sub=Coprocessor.Master.CP_org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver 2023-07-22T12:25:56,314 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering Master,sub=Coprocessor.Master.CP_org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver Metrics about HBase MasterObservers 2023-07-22T12:25:56,315 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:25:56,315 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-07-22T12:25:56,315 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-07-22T12:25:56,315 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-07-22T12:25:56,315 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-07-22T12:25:56,315 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-07-22T12:25:56,460 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-07-22T12:25:56,489 INFO [Thread-249 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:38703 2023-07-22T12:25:56,922 WARN [Listener at jenkins-hbase3.apache.org/42589 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:25:57,005 WARN [Listener at jenkins-hbase3.apache.org/42589 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-07-22T12:25:57,005 INFO [Listener at jenkins-hbase3.apache.org/42589 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:25:57,011 INFO [Listener at jenkins-hbase3.apache.org/42589 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_42259_cluster____.gix2b9/webapp 2023-07-22T12:25:57,080 INFO [Listener at jenkins-hbase3.apache.org/42589 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-07-22T12:25:57,080 INFO [Listener at jenkins-hbase3.apache.org/42589 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-07-22T12:25:57,080 INFO [Listener at jenkins-hbase3.apache.org/42589 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-07-22T12:25:57,080 INFO [Listener at jenkins-hbase3.apache.org/42589 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-07-22T12:25:57,104 INFO [Listener at jenkins-hbase3.apache.org/42589 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:25:57,305 INFO [Listener at jenkins-hbase3.apache.org/42589 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:25:57,383 INFO [Listener at jenkins-hbase3.apache.org/42589 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:25:57,390 INFO [Listener at jenkins-hbase3.apache.org/42589 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:42259 2023-07-22T12:25:57,612 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:25:57,693 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-07-22T12:25:57,699 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-07-22T12:25:57,802 WARN [Listener at jenkins-hbase3.apache.org/35597 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:25:57,827 WARN [Listener at jenkins-hbase3.apache.org/35597 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-07-22T12:25:57,828 INFO [Listener at jenkins-hbase3.apache.org/35597 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:25:57,834 INFO [Listener at jenkins-hbase3.apache.org/35597 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_42319_node____.qpf3ic/webapp 2023-07-22T12:25:57,889 INFO [Listener at jenkins-hbase3.apache.org/35597 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-07-22T12:25:57,889 INFO [Listener at jenkins-hbase3.apache.org/35597 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-07-22T12:25:57,889 INFO [Listener at jenkins-hbase3.apache.org/35597 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-07-22T12:25:57,889 INFO [Listener at jenkins-hbase3.apache.org/35597 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-07-22T12:25:57,899 INFO [Listener at jenkins-hbase3.apache.org/35597 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:25:57,922 INFO [Listener at jenkins-hbase3.apache.org/35597 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:25:58,020 INFO [Listener at jenkins-hbase3.apache.org/35597 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:25:58,034 INFO [Listener at jenkins-hbase3.apache.org/35597 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:42319 2023-07-22T12:25:58,214 WARN [Listener at jenkins-hbase3.apache.org/43519 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:25:58,223 WARN [Listener at jenkins-hbase3.apache.org/43519 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-07-22T12:25:58,224 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:25:58,234 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_34711_node____vnbflh/webapp 2023-07-22T12:25:58,302 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-07-22T12:25:58,302 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-07-22T12:25:58,303 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-07-22T12:25:58,303 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-07-22T12:25:58,314 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:25:58,340 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:25:58,446 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:25:58,457 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:34711 2023-07-22T12:25:58,503 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-07-22T12:25:58,503 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:35695/backupUT 2023-07-22T12:25:58,516 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:25:58,521 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:46424, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:25:58,530 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns1'} 2023-07-22T12:25:58,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure2.ProcedureExecutor(1033): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-07-22T12:25:58,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=9 2023-07-22T12:25:58,562 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:25:58,567 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 34 msec 2023-07-22T12:25:58,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=9 2023-07-22T12:25:58,658 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns2'} 2023-07-22T12:25:58,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure2.ProcedureExecutor(1033): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-07-22T12:25:58,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=10 2023-07-22T12:25:58,672 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:25:58,678 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 18 msec 2023-07-22T12:25:58,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=10 2023-07-22T12:25:58,769 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns3'} 2023-07-22T12:25:58,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure2.ProcedureExecutor(1033): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-07-22T12:25:58,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=11 2023-07-22T12:25:58,786 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:25:58,792 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 21 msec 2023-07-22T12:25:58,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=11 2023-07-22T12:25:58,881 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns4'} 2023-07-22T12:25:58,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure2.ProcedureExecutor(1033): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-07-22T12:25:58,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=12 2023-07-22T12:25:58,894 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:25:58,899 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 16 msec 2023-07-22T12:25:58,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=12 2023-07-22T12:25:59,005 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'test-1690028758503', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:25:59,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure2.ProcedureExecutor(1033): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1690028758503 2023-07-22T12:25:59,010 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1690028758503 execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:25:59,010 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "test-1690028758503" procId is: 13 2023-07-22T12:25:59,011 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:25:59,012 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1690028758503 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:25:59,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:25:59,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:25:59,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:25:59,460 INFO [RegionOpenAndInit-test-1690028758503-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 648e0182e924cbe38699c76e79cee0a5, NAME => 'test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1690028758503', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb 2023-07-22T12:25:59,479 DEBUG [RegionOpenAndInit-test-1690028758503-pool-0 {}] regionserver.HRegion(888): Instantiated test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:25:59,479 DEBUG [RegionOpenAndInit-test-1690028758503-pool-0 {}] regionserver.HRegion(1668): Closing 648e0182e924cbe38699c76e79cee0a5, disabling compactions & flushes 2023-07-22T12:25:59,479 INFO [RegionOpenAndInit-test-1690028758503-pool-0 {}] regionserver.HRegion(1690): Closing region test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5. 2023-07-22T12:25:59,479 DEBUG [RegionOpenAndInit-test-1690028758503-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5. 2023-07-22T12:25:59,479 DEBUG [RegionOpenAndInit-test-1690028758503-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5. after waiting 0 ms 2023-07-22T12:25:59,479 DEBUG [RegionOpenAndInit-test-1690028758503-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5. 2023-07-22T12:25:59,479 INFO [RegionOpenAndInit-test-1690028758503-pool-0 {}] regionserver.HRegion(1909): Closed test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5. 2023-07-22T12:25:59,479 DEBUG [RegionOpenAndInit-test-1690028758503-pool-0 {}] regionserver.HRegion(1622): Region close journal for 648e0182e924cbe38699c76e79cee0a5: 2023-07-22T12:25:59,481 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1690028758503 execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:25:59,481 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1690028759481"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690028759481"}]},"ts":"1690028759481"} 2023-07-22T12:25:59,484 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:25:59,485 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1690028758503 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:25:59,485 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1690028758503","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028759485"}]},"ts":"1690028759485"} 2023-07-22T12:25:59,487 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1690028758503, state=ENABLING in hbase:meta 2023-07-22T12:25:59,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1690028758503, region=648e0182e924cbe38699c76e79cee0a5, ASSIGN}] 2023-07-22T12:25:59,495 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1690028758503, region=648e0182e924cbe38699c76e79cee0a5, ASSIGN 2023-07-22T12:25:59,496 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1690028758503, region=648e0182e924cbe38699c76e79cee0a5, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45275,1690028749134; forceNewPlan=false, retain=false 2023-07-22T12:25:59,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:25:59,648 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=648e0182e924cbe38699c76e79cee0a5, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:25:59,651 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure 648e0182e924cbe38699c76e79cee0a5, server=jenkins-hbase3.apache.org,45275,1690028749134}] 2023-07-22T12:25:59,814 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5. 2023-07-22T12:25:59,814 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => 648e0182e924cbe38699c76e79cee0a5, NAME => 'test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:25:59,815 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:25:59,815 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1690028758503 648e0182e924cbe38699c76e79cee0a5 2023-07-22T12:25:59,815 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:25:59,815 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for 648e0182e924cbe38699c76e79cee0a5 2023-07-22T12:25:59,815 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for 648e0182e924cbe38699c76e79cee0a5 2023-07-22T12:25:59,819 INFO [StoreOpener-648e0182e924cbe38699c76e79cee0a5-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 648e0182e924cbe38699c76e79cee0a5 2023-07-22T12:25:59,821 INFO [StoreOpener-648e0182e924cbe38699c76e79cee0a5-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 648e0182e924cbe38699c76e79cee0a5 columnFamilyName f 2023-07-22T12:25:59,821 DEBUG [StoreOpener-648e0182e924cbe38699c76e79cee0a5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:25:59,823 INFO [StoreOpener-648e0182e924cbe38699c76e79cee0a5-1 {}] regionserver.HStore(324): Store=648e0182e924cbe38699c76e79cee0a5/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:25:59,826 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/default/test-1690028758503/648e0182e924cbe38699c76e79cee0a5 2023-07-22T12:25:59,827 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/default/test-1690028758503/648e0182e924cbe38699c76e79cee0a5 2023-07-22T12:25:59,834 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for 648e0182e924cbe38699c76e79cee0a5 2023-07-22T12:25:59,842 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/default/test-1690028758503/648e0182e924cbe38699c76e79cee0a5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:25:59,843 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened 648e0182e924cbe38699c76e79cee0a5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9404450080, jitterRate=-0.12414233386516571}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:25:59,844 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for 648e0182e924cbe38699c76e79cee0a5: 2023-07-22T12:25:59,846 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2552): Post open deploy tasks for test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5., pid=15, masterSystemTime=1690028759804 2023-07-22T12:25:59,849 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2579): Finished post open deploy task for test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5. 2023-07-22T12:25:59,849 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5. 2023-07-22T12:25:59,850 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=648e0182e924cbe38699c76e79cee0a5, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:25:59,856 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=15, resume processing ppid=14 2023-07-22T12:25:59,856 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure 648e0182e924cbe38699c76e79cee0a5, server=jenkins-hbase3.apache.org,45275,1690028749134 in 202 msec 2023-07-22T12:25:59,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=14, resume processing ppid=13 2023-07-22T12:25:59,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1690028758503, region=648e0182e924cbe38699c76e79cee0a5, ASSIGN in 363 msec 2023-07-22T12:25:59,861 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1690028758503 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:25:59,861 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1690028758503","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028759861"}]},"ts":"1690028759861"} 2023-07-22T12:25:59,863 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1690028758503, state=ENABLED in hbase:meta 2023-07-22T12:25:59,866 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1690028758503 execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:25:59,868 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1690028758503 in 861 msec 2023-07-22T12:26:00,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:26:00,129 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: default:test-1690028758503, procId: 13 completed 2023-07-22T12:26:00,130 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x105cf204 to 127.0.0.1:60489 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:26:00,136 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@676720bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:26:00,138 DEBUG [hconnection-0x78a6b326-metaLookup-shared--pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:26:00,141 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:46796, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:26:00,154 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45275 {}] regionserver.HRegion(8210): writing data to region test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5. with WAL disabled. Data may be lost in the event of a crash. 2023-07-22T12:26:00,257 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns2:test-16900287585031', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:26:00,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure2.ProcedureExecutor(1033): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-16900287585031 2023-07-22T12:26:00,261 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16900287585031 execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:26:00,261 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:26:00,261 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "test-16900287585031" procId is: 16 2023-07-22T12:26:00,263 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-16900287585031 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:26:00,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-07-22T12:26:00,318 INFO [RegionOpenAndInit-ns2:test-16900287585031-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 40f3d448196e452f47f760cf6a94ef80, NAME => 'ns2:test-16900287585031,,1690028760257.40f3d448196e452f47f760cf6a94ef80.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16900287585031', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb 2023-07-22T12:26:00,340 DEBUG [RegionOpenAndInit-ns2:test-16900287585031-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16900287585031,,1690028760257.40f3d448196e452f47f760cf6a94ef80.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:26:00,341 DEBUG [RegionOpenAndInit-ns2:test-16900287585031-pool-0 {}] regionserver.HRegion(1668): Closing 40f3d448196e452f47f760cf6a94ef80, disabling compactions & flushes 2023-07-22T12:26:00,341 INFO [RegionOpenAndInit-ns2:test-16900287585031-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16900287585031,,1690028760257.40f3d448196e452f47f760cf6a94ef80. 2023-07-22T12:26:00,341 DEBUG [RegionOpenAndInit-ns2:test-16900287585031-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16900287585031,,1690028760257.40f3d448196e452f47f760cf6a94ef80. 2023-07-22T12:26:00,341 DEBUG [RegionOpenAndInit-ns2:test-16900287585031-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16900287585031,,1690028760257.40f3d448196e452f47f760cf6a94ef80. after waiting 0 ms 2023-07-22T12:26:00,341 DEBUG [RegionOpenAndInit-ns2:test-16900287585031-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16900287585031,,1690028760257.40f3d448196e452f47f760cf6a94ef80. 2023-07-22T12:26:00,341 INFO [RegionOpenAndInit-ns2:test-16900287585031-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16900287585031,,1690028760257.40f3d448196e452f47f760cf6a94ef80. 2023-07-22T12:26:00,341 DEBUG [RegionOpenAndInit-ns2:test-16900287585031-pool-0 {}] regionserver.HRegion(1622): Region close journal for 40f3d448196e452f47f760cf6a94ef80: 2023-07-22T12:26:00,343 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-16900287585031 execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:26:00,343 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16900287585031,,1690028760257.40f3d448196e452f47f760cf6a94ef80.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1690028760343"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690028760343"}]},"ts":"1690028760343"} 2023-07-22T12:26:00,346 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:26:00,347 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-16900287585031 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:26:00,348 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16900287585031","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028760347"}]},"ts":"1690028760347"} 2023-07-22T12:26:00,350 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16900287585031, state=ENABLING in hbase:meta 2023-07-22T12:26:00,356 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16900287585031, region=40f3d448196e452f47f760cf6a94ef80, ASSIGN}] 2023-07-22T12:26:00,359 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16900287585031, region=40f3d448196e452f47f760cf6a94ef80, ASSIGN 2023-07-22T12:26:00,361 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16900287585031, region=40f3d448196e452f47f760cf6a94ef80, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45275,1690028749134; forceNewPlan=false, retain=false 2023-07-22T12:26:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-07-22T12:26:00,513 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=40f3d448196e452f47f760cf6a94ef80, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:00,516 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure 40f3d448196e452f47f760cf6a94ef80, server=jenkins-hbase3.apache.org,45275,1690028749134}] 2023-07-22T12:26:00,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-07-22T12:26:00,674 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-16900287585031,,1690028760257.40f3d448196e452f47f760cf6a94ef80. 2023-07-22T12:26:00,674 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => 40f3d448196e452f47f760cf6a94ef80, NAME => 'ns2:test-16900287585031,,1690028760257.40f3d448196e452f47f760cf6a94ef80.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:26:00,675 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:26:00,675 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16900287585031 40f3d448196e452f47f760cf6a94ef80 2023-07-22T12:26:00,675 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-16900287585031,,1690028760257.40f3d448196e452f47f760cf6a94ef80.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:26:00,675 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for 40f3d448196e452f47f760cf6a94ef80 2023-07-22T12:26:00,675 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for 40f3d448196e452f47f760cf6a94ef80 2023-07-22T12:26:00,679 INFO [StoreOpener-40f3d448196e452f47f760cf6a94ef80-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 40f3d448196e452f47f760cf6a94ef80 2023-07-22T12:26:00,682 INFO [StoreOpener-40f3d448196e452f47f760cf6a94ef80-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40f3d448196e452f47f760cf6a94ef80 columnFamilyName f 2023-07-22T12:26:00,682 DEBUG [StoreOpener-40f3d448196e452f47f760cf6a94ef80-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:26:00,683 INFO [StoreOpener-40f3d448196e452f47f760cf6a94ef80-1 {}] regionserver.HStore(324): Store=40f3d448196e452f47f760cf6a94ef80/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:26:00,684 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/ns2/test-16900287585031/40f3d448196e452f47f760cf6a94ef80 2023-07-22T12:26:00,684 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/ns2/test-16900287585031/40f3d448196e452f47f760cf6a94ef80 2023-07-22T12:26:00,688 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for 40f3d448196e452f47f760cf6a94ef80 2023-07-22T12:26:00,697 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/ns2/test-16900287585031/40f3d448196e452f47f760cf6a94ef80/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:26:00,698 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened 40f3d448196e452f47f760cf6a94ef80; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9554456800, jitterRate=-0.11017186939716339}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:26:00,699 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for 40f3d448196e452f47f760cf6a94ef80: 2023-07-22T12:26:00,700 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2552): Post open deploy tasks for ns2:test-16900287585031,,1690028760257.40f3d448196e452f47f760cf6a94ef80., pid=18, masterSystemTime=1690028760669 2023-07-22T12:26:00,702 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2579): Finished post open deploy task for ns2:test-16900287585031,,1690028760257.40f3d448196e452f47f760cf6a94ef80. 2023-07-22T12:26:00,702 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-16900287585031,,1690028760257.40f3d448196e452f47f760cf6a94ef80. 2023-07-22T12:26:00,703 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=40f3d448196e452f47f760cf6a94ef80, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:00,709 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=18, resume processing ppid=17 2023-07-22T12:26:00,709 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure 40f3d448196e452f47f760cf6a94ef80, server=jenkins-hbase3.apache.org,45275,1690028749134 in 190 msec 2023-07-22T12:26:00,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=17, resume processing ppid=16 2023-07-22T12:26:00,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16900287585031, region=40f3d448196e452f47f760cf6a94ef80, ASSIGN in 353 msec 2023-07-22T12:26:00,713 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-16900287585031 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:26:00,713 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16900287585031","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028760713"}]},"ts":"1690028760713"} 2023-07-22T12:26:00,715 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16900287585031, state=ENABLED in hbase:meta 2023-07-22T12:26:00,717 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16900287585031 execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:26:00,719 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-16900287585031 in 459 msec 2023-07-22T12:26:00,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-07-22T12:26:00,870 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns2:test-16900287585031, procId: 16 completed 2023-07-22T12:26:00,874 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45275 {}] regionserver.HRegion(8210): writing data to region ns2:test-16900287585031,,1690028760257.40f3d448196e452f47f760cf6a94ef80. with WAL disabled. Data may be lost in the event of a crash. 2023-07-22T12:26:00,985 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns3:test-16900287585032', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:26:00,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure2.ProcedureExecutor(1033): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-16900287585032 2023-07-22T12:26:00,990 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16900287585032 execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:26:00,990 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:26:00,990 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "test-16900287585032" procId is: 19 2023-07-22T12:26:00,991 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-16900287585032 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:26:00,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:26:01,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:26:01,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:26:01,418 INFO [RegionOpenAndInit-ns3:test-16900287585032-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 01adde20a336e8a75d65a8603dd7f6c0, NAME => 'ns3:test-16900287585032,,1690028760985.01adde20a336e8a75d65a8603dd7f6c0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16900287585032', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb 2023-07-22T12:26:01,437 DEBUG [RegionOpenAndInit-ns3:test-16900287585032-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16900287585032,,1690028760985.01adde20a336e8a75d65a8603dd7f6c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:26:01,437 DEBUG [RegionOpenAndInit-ns3:test-16900287585032-pool-0 {}] regionserver.HRegion(1668): Closing 01adde20a336e8a75d65a8603dd7f6c0, disabling compactions & flushes 2023-07-22T12:26:01,437 INFO [RegionOpenAndInit-ns3:test-16900287585032-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16900287585032,,1690028760985.01adde20a336e8a75d65a8603dd7f6c0. 2023-07-22T12:26:01,437 DEBUG [RegionOpenAndInit-ns3:test-16900287585032-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16900287585032,,1690028760985.01adde20a336e8a75d65a8603dd7f6c0. 2023-07-22T12:26:01,437 DEBUG [RegionOpenAndInit-ns3:test-16900287585032-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16900287585032,,1690028760985.01adde20a336e8a75d65a8603dd7f6c0. after waiting 0 ms 2023-07-22T12:26:01,437 DEBUG [RegionOpenAndInit-ns3:test-16900287585032-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16900287585032,,1690028760985.01adde20a336e8a75d65a8603dd7f6c0. 2023-07-22T12:26:01,437 INFO [RegionOpenAndInit-ns3:test-16900287585032-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16900287585032,,1690028760985.01adde20a336e8a75d65a8603dd7f6c0. 2023-07-22T12:26:01,438 DEBUG [RegionOpenAndInit-ns3:test-16900287585032-pool-0 {}] regionserver.HRegion(1622): Region close journal for 01adde20a336e8a75d65a8603dd7f6c0: 2023-07-22T12:26:01,439 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-16900287585032 execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:26:01,440 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16900287585032,,1690028760985.01adde20a336e8a75d65a8603dd7f6c0.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1690028761439"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690028761439"}]},"ts":"1690028761439"} 2023-07-22T12:26:01,442 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:26:01,443 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-16900287585032 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:26:01,444 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16900287585032","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028761444"}]},"ts":"1690028761444"} 2023-07-22T12:26:01,446 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16900287585032, state=ENABLING in hbase:meta 2023-07-22T12:26:01,450 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16900287585032, region=01adde20a336e8a75d65a8603dd7f6c0, ASSIGN}] 2023-07-22T12:26:01,452 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16900287585032, region=01adde20a336e8a75d65a8603dd7f6c0, ASSIGN 2023-07-22T12:26:01,453 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16900287585032, region=01adde20a336e8a75d65a8603dd7f6c0, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45275,1690028749134; forceNewPlan=false, retain=false 2023-07-22T12:26:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:26:01,604 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=01adde20a336e8a75d65a8603dd7f6c0, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:01,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure 01adde20a336e8a75d65a8603dd7f6c0, server=jenkins-hbase3.apache.org,45275,1690028749134}] 2023-07-22T12:26:01,765 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-16900287585032,,1690028760985.01adde20a336e8a75d65a8603dd7f6c0. 2023-07-22T12:26:01,765 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => 01adde20a336e8a75d65a8603dd7f6c0, NAME => 'ns3:test-16900287585032,,1690028760985.01adde20a336e8a75d65a8603dd7f6c0.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:26:01,765 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:26:01,766 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16900287585032 01adde20a336e8a75d65a8603dd7f6c0 2023-07-22T12:26:01,766 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-16900287585032,,1690028760985.01adde20a336e8a75d65a8603dd7f6c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:26:01,766 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for 01adde20a336e8a75d65a8603dd7f6c0 2023-07-22T12:26:01,766 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for 01adde20a336e8a75d65a8603dd7f6c0 2023-07-22T12:26:01,768 INFO [StoreOpener-01adde20a336e8a75d65a8603dd7f6c0-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 01adde20a336e8a75d65a8603dd7f6c0 2023-07-22T12:26:01,770 INFO [StoreOpener-01adde20a336e8a75d65a8603dd7f6c0-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01adde20a336e8a75d65a8603dd7f6c0 columnFamilyName f 2023-07-22T12:26:01,770 DEBUG [StoreOpener-01adde20a336e8a75d65a8603dd7f6c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:26:01,770 INFO [StoreOpener-01adde20a336e8a75d65a8603dd7f6c0-1 {}] regionserver.HStore(324): Store=01adde20a336e8a75d65a8603dd7f6c0/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:26:01,772 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/ns3/test-16900287585032/01adde20a336e8a75d65a8603dd7f6c0 2023-07-22T12:26:01,773 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/ns3/test-16900287585032/01adde20a336e8a75d65a8603dd7f6c0 2023-07-22T12:26:01,777 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for 01adde20a336e8a75d65a8603dd7f6c0 2023-07-22T12:26:01,780 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/ns3/test-16900287585032/01adde20a336e8a75d65a8603dd7f6c0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:26:01,781 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened 01adde20a336e8a75d65a8603dd7f6c0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11179938240, jitterRate=0.041212886571884155}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:26:01,781 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for 01adde20a336e8a75d65a8603dd7f6c0: 2023-07-22T12:26:01,782 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2552): Post open deploy tasks for ns3:test-16900287585032,,1690028760985.01adde20a336e8a75d65a8603dd7f6c0., pid=21, masterSystemTime=1690028761759 2023-07-22T12:26:01,785 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2579): Finished post open deploy task for ns3:test-16900287585032,,1690028760985.01adde20a336e8a75d65a8603dd7f6c0. 2023-07-22T12:26:01,785 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-16900287585032,,1690028760985.01adde20a336e8a75d65a8603dd7f6c0. 2023-07-22T12:26:01,786 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=01adde20a336e8a75d65a8603dd7f6c0, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:01,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=21, resume processing ppid=20 2023-07-22T12:26:01,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure 01adde20a336e8a75d65a8603dd7f6c0, server=jenkins-hbase3.apache.org,45275,1690028749134 in 181 msec 2023-07-22T12:26:01,794 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=20, resume processing ppid=19 2023-07-22T12:26:01,794 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16900287585032, region=01adde20a336e8a75d65a8603dd7f6c0, ASSIGN in 341 msec 2023-07-22T12:26:01,795 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-16900287585032 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:26:01,795 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16900287585032","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028761795"}]},"ts":"1690028761795"} 2023-07-22T12:26:01,797 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16900287585032, state=ENABLED in hbase:meta 2023-07-22T12:26:01,800 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16900287585032 execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:26:01,802 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-16900287585032 in 815 msec 2023-07-22T12:26:02,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:26:02,099 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns3:test-16900287585032, procId: 19 completed 2023-07-22T12:26:02,100 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-16900287585032 get assigned. Timeout = 60000ms 2023-07-22T12:26:02,102 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-07-22T12:26:02,106 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-16900287585032 assigned to meta. Checking AM states. 2023-07-22T12:26:02,107 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-07-22T12:26:02,107 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-16900287585032 assigned. 2023-07-22T12:26:02,109 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns4:test-16900287585033', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:26:02,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure2.ProcedureExecutor(1033): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-16900287585033 2023-07-22T12:26:02,113 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16900287585033 execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:26:02,114 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:26:02,114 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns4" qualifier: "test-16900287585033" procId is: 22 2023-07-22T12:26:02,115 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-16900287585033 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:26:02,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:26:02,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:26:02,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:26:02,537 INFO [RegionOpenAndInit-ns4:test-16900287585033-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => ed8e6284cfcb2ae38b4da50cf240aee9, NAME => 'ns4:test-16900287585033,,1690028762109.ed8e6284cfcb2ae38b4da50cf240aee9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-16900287585033', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb 2023-07-22T12:26:02,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:26:02,954 DEBUG [RegionOpenAndInit-ns4:test-16900287585033-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-16900287585033,,1690028762109.ed8e6284cfcb2ae38b4da50cf240aee9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:26:02,954 DEBUG [RegionOpenAndInit-ns4:test-16900287585033-pool-0 {}] regionserver.HRegion(1668): Closing ed8e6284cfcb2ae38b4da50cf240aee9, disabling compactions & flushes 2023-07-22T12:26:02,954 INFO [RegionOpenAndInit-ns4:test-16900287585033-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-16900287585033,,1690028762109.ed8e6284cfcb2ae38b4da50cf240aee9. 2023-07-22T12:26:02,954 DEBUG [RegionOpenAndInit-ns4:test-16900287585033-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16900287585033,,1690028762109.ed8e6284cfcb2ae38b4da50cf240aee9. 2023-07-22T12:26:02,954 DEBUG [RegionOpenAndInit-ns4:test-16900287585033-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16900287585033,,1690028762109.ed8e6284cfcb2ae38b4da50cf240aee9. after waiting 0 ms 2023-07-22T12:26:02,954 DEBUG [RegionOpenAndInit-ns4:test-16900287585033-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16900287585033,,1690028762109.ed8e6284cfcb2ae38b4da50cf240aee9. 2023-07-22T12:26:02,954 INFO [RegionOpenAndInit-ns4:test-16900287585033-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-16900287585033,,1690028762109.ed8e6284cfcb2ae38b4da50cf240aee9. 2023-07-22T12:26:02,954 DEBUG [RegionOpenAndInit-ns4:test-16900287585033-pool-0 {}] regionserver.HRegion(1622): Region close journal for ed8e6284cfcb2ae38b4da50cf240aee9: 2023-07-22T12:26:02,956 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-16900287585033 execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:26:02,956 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-16900287585033,,1690028762109.ed8e6284cfcb2ae38b4da50cf240aee9.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1690028762956"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690028762956"}]},"ts":"1690028762956"} 2023-07-22T12:26:02,958 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:26:02,959 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-16900287585033 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:26:02,959 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16900287585033","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028762959"}]},"ts":"1690028762959"} 2023-07-22T12:26:02,961 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16900287585033, state=ENABLING in hbase:meta 2023-07-22T12:26:02,966 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16900287585033, region=ed8e6284cfcb2ae38b4da50cf240aee9, ASSIGN}] 2023-07-22T12:26:02,968 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16900287585033, region=ed8e6284cfcb2ae38b4da50cf240aee9, ASSIGN 2023-07-22T12:26:02,969 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-16900287585033, region=ed8e6284cfcb2ae38b4da50cf240aee9, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45275,1690028749134; forceNewPlan=false, retain=false 2023-07-22T12:26:03,120 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=ed8e6284cfcb2ae38b4da50cf240aee9, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:03,123 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure ed8e6284cfcb2ae38b4da50cf240aee9, server=jenkins-hbase3.apache.org,45275,1690028749134}] 2023-07-22T12:26:03,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:26:03,279 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-16900287585033,,1690028762109.ed8e6284cfcb2ae38b4da50cf240aee9. 2023-07-22T12:26:03,280 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => ed8e6284cfcb2ae38b4da50cf240aee9, NAME => 'ns4:test-16900287585033,,1690028762109.ed8e6284cfcb2ae38b4da50cf240aee9.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:26:03,280 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:26:03,280 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16900287585033 ed8e6284cfcb2ae38b4da50cf240aee9 2023-07-22T12:26:03,280 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-16900287585033,,1690028762109.ed8e6284cfcb2ae38b4da50cf240aee9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:26:03,281 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for ed8e6284cfcb2ae38b4da50cf240aee9 2023-07-22T12:26:03,281 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for ed8e6284cfcb2ae38b4da50cf240aee9 2023-07-22T12:26:03,282 INFO [StoreOpener-ed8e6284cfcb2ae38b4da50cf240aee9-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region ed8e6284cfcb2ae38b4da50cf240aee9 2023-07-22T12:26:03,285 INFO [StoreOpener-ed8e6284cfcb2ae38b4da50cf240aee9-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ed8e6284cfcb2ae38b4da50cf240aee9 columnFamilyName f 2023-07-22T12:26:03,285 DEBUG [StoreOpener-ed8e6284cfcb2ae38b4da50cf240aee9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:26:03,285 INFO [StoreOpener-ed8e6284cfcb2ae38b4da50cf240aee9-1 {}] regionserver.HStore(324): Store=ed8e6284cfcb2ae38b4da50cf240aee9/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:26:03,287 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/ns4/test-16900287585033/ed8e6284cfcb2ae38b4da50cf240aee9 2023-07-22T12:26:03,287 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/ns4/test-16900287585033/ed8e6284cfcb2ae38b4da50cf240aee9 2023-07-22T12:26:03,291 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for ed8e6284cfcb2ae38b4da50cf240aee9 2023-07-22T12:26:03,294 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/ns4/test-16900287585033/ed8e6284cfcb2ae38b4da50cf240aee9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:26:03,295 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened ed8e6284cfcb2ae38b4da50cf240aee9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10911966560, jitterRate=0.016256079077720642}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:26:03,296 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for ed8e6284cfcb2ae38b4da50cf240aee9: 2023-07-22T12:26:03,297 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2552): Post open deploy tasks for ns4:test-16900287585033,,1690028762109.ed8e6284cfcb2ae38b4da50cf240aee9., pid=24, masterSystemTime=1690028763275 2023-07-22T12:26:03,299 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2579): Finished post open deploy task for ns4:test-16900287585033,,1690028762109.ed8e6284cfcb2ae38b4da50cf240aee9. 2023-07-22T12:26:03,299 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-16900287585033,,1690028762109.ed8e6284cfcb2ae38b4da50cf240aee9. 2023-07-22T12:26:03,300 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=ed8e6284cfcb2ae38b4da50cf240aee9, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:03,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=24, resume processing ppid=23 2023-07-22T12:26:03,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure ed8e6284cfcb2ae38b4da50cf240aee9, server=jenkins-hbase3.apache.org,45275,1690028749134 in 179 msec 2023-07-22T12:26:03,307 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=23, resume processing ppid=22 2023-07-22T12:26:03,307 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-16900287585033, region=ed8e6284cfcb2ae38b4da50cf240aee9, ASSIGN in 340 msec 2023-07-22T12:26:03,308 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-16900287585033 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:26:03,309 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16900287585033","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028763308"}]},"ts":"1690028763308"} 2023-07-22T12:26:03,310 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16900287585033, state=ENABLED in hbase:meta 2023-07-22T12:26:03,313 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16900287585033 execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:26:03,315 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-16900287585033 in 1.2040 sec 2023-07-22T12:26:03,700 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-16900287585033' 2023-07-22T12:26:03,700 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1690028758503' 2023-07-22T12:26:03,701 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16900287585032' 2023-07-22T12:26:03,701 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-16900287585031' 2023-07-22T12:26:04,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:26:04,224 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns4:test-16900287585033, procId: 22 completed 2023-07-22T12:26:04,224 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-16900287585033 get assigned. Timeout = 60000ms 2023-07-22T12:26:04,224 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-07-22T12:26:04,229 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-16900287585033 assigned to meta. Checking AM states. 2023-07-22T12:26:04,229 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-07-22T12:26:04,229 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-16900287585033 assigned. 2023-07-22T12:26:04,230 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x105cf204 to 127.0.0.1:60489 2023-07-22T12:26:04,230 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:26:04,268 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] hbase.ResourceChecker(147): before: backup.TestBackupDeleteWithFailures#testBackupDeleteWithFailures Thread=583, OpenFileDescriptor=660, MaxFileDescriptor=60000, SystemLoadAverage=202, ProcessCount=169, AvailableMemoryMB=17309 2023-07-22T12:26:04,269 WARN [Listener at jenkins-hbase3.apache.org/43519 {}] hbase.ResourceChecker(130): Thread=583 is superior to 500 2023-07-22T12:26:04,271 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] backup.TestBackupDeleteWithFailures(143): test repair backup delete on a single table with data and failures PRE_DELETE_SNAPSHOT_FAILURE 2023-07-22T12:26:04,271 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4c0443b7 to 127.0.0.1:60489 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:26:04,277 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7749a8b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:26:04,285 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:26:04,287 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:48386, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:26:04,298 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:26:04,300 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:45140, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:26:04,310 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'backup'} 2023-07-22T12:26:04,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure2.ProcedureExecutor(1033): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-07-22T12:26:04,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=25 2023-07-22T12:26:04,328 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:26:04,333 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=25, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 21 msec 2023-07-22T12:26:04,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=25 2023-07-22T12:26:04,433 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:26:04,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure2.ProcedureExecutor(1033): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-07-22T12:26:04,438 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:26:04,438 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:26:04,438 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2023-07-22T12:26:04,439 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:26:04,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-07-22T12:26:04,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-07-22T12:26:04,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-07-22T12:26:04,859 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 73a8d9b14b06f35bfa58d9fccd108576, NAME => 'backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb 2023-07-22T12:26:05,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-07-22T12:26:05,275 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:26:05,275 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing 73a8d9b14b06f35bfa58d9fccd108576, disabling compactions & flushes 2023-07-22T12:26:05,276 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:05,276 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:05,276 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. after waiting 0 ms 2023-07-22T12:26:05,276 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:05,276 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:05,276 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for 73a8d9b14b06f35bfa58d9fccd108576: 2023-07-22T12:26:05,278 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:26:05,278 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1690028765278"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690028765278"}]},"ts":"1690028765278"} 2023-07-22T12:26:05,280 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:26:05,281 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:26:05,282 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028765281"}]},"ts":"1690028765281"} 2023-07-22T12:26:05,284 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-07-22T12:26:05,289 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=73a8d9b14b06f35bfa58d9fccd108576, ASSIGN}] 2023-07-22T12:26:05,291 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=73a8d9b14b06f35bfa58d9fccd108576, ASSIGN 2023-07-22T12:26:05,292 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=73a8d9b14b06f35bfa58d9fccd108576, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45275,1690028749134; forceNewPlan=false, retain=false 2023-07-22T12:26:05,324 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:26:05,443 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=73a8d9b14b06f35bfa58d9fccd108576, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:05,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; OpenRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576, server=jenkins-hbase3.apache.org,45275,1690028749134}] 2023-07-22T12:26:05,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-07-22T12:26:05,606 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(130): Open backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:05,606 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7254): Opening region: {ENCODED => 73a8d9b14b06f35bfa58d9fccd108576, NAME => 'backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:26:05,607 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:26:05,607 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:05,607 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(888): Instantiated backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:26:05,607 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7296): checking encryption for 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:05,608 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7299): checking classloading for 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:05,610 INFO [StoreOpener-73a8d9b14b06f35bfa58d9fccd108576-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:05,612 INFO [StoreOpener-73a8d9b14b06f35bfa58d9fccd108576-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73a8d9b14b06f35bfa58d9fccd108576 columnFamilyName meta 2023-07-22T12:26:05,612 DEBUG [StoreOpener-73a8d9b14b06f35bfa58d9fccd108576-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:26:05,613 INFO [StoreOpener-73a8d9b14b06f35bfa58d9fccd108576-1 {}] regionserver.HStore(324): Store=73a8d9b14b06f35bfa58d9fccd108576/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:26:05,613 INFO [StoreOpener-73a8d9b14b06f35bfa58d9fccd108576-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:05,615 INFO [StoreOpener-73a8d9b14b06f35bfa58d9fccd108576-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73a8d9b14b06f35bfa58d9fccd108576 columnFamilyName session 2023-07-22T12:26:05,615 DEBUG [StoreOpener-73a8d9b14b06f35bfa58d9fccd108576-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:26:05,616 INFO [StoreOpener-73a8d9b14b06f35bfa58d9fccd108576-1 {}] regionserver.HStore(324): Store=73a8d9b14b06f35bfa58d9fccd108576/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:26:05,617 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:05,617 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:05,619 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-07-22T12:26:05,621 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1076): writing seq id for 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:05,624 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:26:05,624 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): Opened 73a8d9b14b06f35bfa58d9fccd108576; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11617591680, jitterRate=0.08197253942489624}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-07-22T12:26:05,625 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(992): Region open journal for 73a8d9b14b06f35bfa58d9fccd108576: 2023-07-22T12:26:05,626 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576., pid=28, masterSystemTime=1690028765600 2023-07-22T12:26:05,628 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:05,628 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(158): Opened backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:05,629 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=73a8d9b14b06f35bfa58d9fccd108576, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:05,633 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=28, resume processing ppid=27 2023-07-22T12:26:05,633 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=28, ppid=27, state=SUCCESS; OpenRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576, server=jenkins-hbase3.apache.org,45275,1690028749134 in 185 msec 2023-07-22T12:26:05,635 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=27, resume processing ppid=26 2023-07-22T12:26:05,635 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=73a8d9b14b06f35bfa58d9fccd108576, ASSIGN in 344 msec 2023-07-22T12:26:05,636 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:26:05,636 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028765636"}]},"ts":"1690028765636"} 2023-07-22T12:26:05,638 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-07-22T12:26:05,641 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:26:05,642 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=26, state=SUCCESS; CreateTableProcedure table=backup:system in 1.2080 sec 2023-07-22T12:26:06,300 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-07-22T12:26:06,300 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-07-22T12:26:06,301 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16900287585031 2023-07-22T12:26:06,301 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-16900287585031 Metrics about Tables on a single HBase RegionServer 2023-07-22T12:26:06,302 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1690028758503 2023-07-22T12:26:06,302 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1690028758503 Metrics about Tables on a single HBase RegionServer 2023-07-22T12:26:06,302 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16900287585032 2023-07-22T12:26:06,303 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-16900287585032 Metrics about Tables on a single HBase RegionServer 2023-07-22T12:26:06,303 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16900287585033 2023-07-22T12:26:06,303 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-16900287585033 Metrics about Tables on a single HBase RegionServer 2023-07-22T12:26:06,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-07-22T12:26:06,549 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: backup:system, procId: 26 completed 2023-07-22T12:26:06,564 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:26:06,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure2.ProcedureExecutor(1033): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-07-22T12:26:06,568 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:26:06,568 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:26:06,568 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2023-07-22T12:26:06,569 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:26:06,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-07-22T12:26:06,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-07-22T12:26:06,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-07-22T12:26:06,984 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 999a9c03f7b1cabe862a9a76f7912808, NAME => 'backup:system_bulk,,1690028766564.999a9c03f7b1cabe862a9a76f7912808.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb 2023-07-22T12:26:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-07-22T12:26:07,398 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1690028766564.999a9c03f7b1cabe862a9a76f7912808.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:26:07,398 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing 999a9c03f7b1cabe862a9a76f7912808, disabling compactions & flushes 2023-07-22T12:26:07,398 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1690028766564.999a9c03f7b1cabe862a9a76f7912808. 2023-07-22T12:26:07,398 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1690028766564.999a9c03f7b1cabe862a9a76f7912808. 2023-07-22T12:26:07,398 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1690028766564.999a9c03f7b1cabe862a9a76f7912808. after waiting 0 ms 2023-07-22T12:26:07,398 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1690028766564.999a9c03f7b1cabe862a9a76f7912808. 2023-07-22T12:26:07,398 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1690028766564.999a9c03f7b1cabe862a9a76f7912808. 2023-07-22T12:26:07,398 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for 999a9c03f7b1cabe862a9a76f7912808: 2023-07-22T12:26:07,400 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:26:07,400 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1690028766564.999a9c03f7b1cabe862a9a76f7912808.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1690028767400"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690028767400"}]},"ts":"1690028767400"} 2023-07-22T12:26:07,402 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:26:07,403 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:26:07,403 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028767403"}]},"ts":"1690028767403"} 2023-07-22T12:26:07,405 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-07-22T12:26:07,410 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=999a9c03f7b1cabe862a9a76f7912808, ASSIGN}] 2023-07-22T12:26:07,412 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=999a9c03f7b1cabe862a9a76f7912808, ASSIGN 2023-07-22T12:26:07,413 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=999a9c03f7b1cabe862a9a76f7912808, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45275,1690028749134; forceNewPlan=false, retain=false 2023-07-22T12:26:07,565 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=999a9c03f7b1cabe862a9a76f7912808, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:07,567 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE; OpenRegionProcedure 999a9c03f7b1cabe862a9a76f7912808, server=jenkins-hbase3.apache.org,45275,1690028749134}] 2023-07-22T12:26:07,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-07-22T12:26:07,725 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1690028766564.999a9c03f7b1cabe862a9a76f7912808. 2023-07-22T12:26:07,725 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7254): Opening region: {ENCODED => 999a9c03f7b1cabe862a9a76f7912808, NAME => 'backup:system_bulk,,1690028766564.999a9c03f7b1cabe862a9a76f7912808.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:26:07,725 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:26:07,725 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk 999a9c03f7b1cabe862a9a76f7912808 2023-07-22T12:26:07,725 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1690028766564.999a9c03f7b1cabe862a9a76f7912808.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:26:07,726 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7296): checking encryption for 999a9c03f7b1cabe862a9a76f7912808 2023-07-22T12:26:07,726 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7299): checking classloading for 999a9c03f7b1cabe862a9a76f7912808 2023-07-22T12:26:07,728 INFO [StoreOpener-999a9c03f7b1cabe862a9a76f7912808-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 999a9c03f7b1cabe862a9a76f7912808 2023-07-22T12:26:07,730 INFO [StoreOpener-999a9c03f7b1cabe862a9a76f7912808-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 999a9c03f7b1cabe862a9a76f7912808 columnFamilyName meta 2023-07-22T12:26:07,730 DEBUG [StoreOpener-999a9c03f7b1cabe862a9a76f7912808-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:26:07,731 INFO [StoreOpener-999a9c03f7b1cabe862a9a76f7912808-1 {}] regionserver.HStore(324): Store=999a9c03f7b1cabe862a9a76f7912808/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:26:07,731 INFO [StoreOpener-999a9c03f7b1cabe862a9a76f7912808-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 999a9c03f7b1cabe862a9a76f7912808 2023-07-22T12:26:07,733 INFO [StoreOpener-999a9c03f7b1cabe862a9a76f7912808-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 999a9c03f7b1cabe862a9a76f7912808 columnFamilyName session 2023-07-22T12:26:07,733 DEBUG [StoreOpener-999a9c03f7b1cabe862a9a76f7912808-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:26:07,733 INFO [StoreOpener-999a9c03f7b1cabe862a9a76f7912808-1 {}] regionserver.HStore(324): Store=999a9c03f7b1cabe862a9a76f7912808/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:26:07,734 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system_bulk/999a9c03f7b1cabe862a9a76f7912808 2023-07-22T12:26:07,735 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system_bulk/999a9c03f7b1cabe862a9a76f7912808 2023-07-22T12:26:07,737 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-07-22T12:26:07,739 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1076): writing seq id for 999a9c03f7b1cabe862a9a76f7912808 2023-07-22T12:26:07,743 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system_bulk/999a9c03f7b1cabe862a9a76f7912808/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:26:07,743 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): Opened 999a9c03f7b1cabe862a9a76f7912808; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9896869120, jitterRate=-0.07828223705291748}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-07-22T12:26:07,744 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(992): Region open journal for 999a9c03f7b1cabe862a9a76f7912808: 2023-07-22T12:26:07,745 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system_bulk,,1690028766564.999a9c03f7b1cabe862a9a76f7912808., pid=31, masterSystemTime=1690028767720 2023-07-22T12:26:07,747 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system_bulk,,1690028766564.999a9c03f7b1cabe862a9a76f7912808. 2023-07-22T12:26:07,748 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1690028766564.999a9c03f7b1cabe862a9a76f7912808. 2023-07-22T12:26:07,748 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=999a9c03f7b1cabe862a9a76f7912808, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:07,752 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=31, resume processing ppid=30 2023-07-22T12:26:07,752 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=31, ppid=30, state=SUCCESS; OpenRegionProcedure 999a9c03f7b1cabe862a9a76f7912808, server=jenkins-hbase3.apache.org,45275,1690028749134 in 183 msec 2023-07-22T12:26:07,754 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=30, resume processing ppid=29 2023-07-22T12:26:07,754 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=30, ppid=29, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=999a9c03f7b1cabe862a9a76f7912808, ASSIGN in 342 msec 2023-07-22T12:26:07,755 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:26:07,755 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028767755"}]},"ts":"1690028767755"} 2023-07-22T12:26:07,757 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-07-22T12:26:07,760 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:26:07,761 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=29, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 1.1960 sec 2023-07-22T12:26:08,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-07-22T12:26:08,678 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: backup:system_bulk, procId: 29 completed 2023-07-22T12:26:08,691 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:26:08,753 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:26:08,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690028768753 (current time:1690028768753). 2023-07-22T12:26:08,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:26:08,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:26:08,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x34916cd6 to 127.0.0.1:60489 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:26:08,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b368f6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:26:08,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:26:08,766 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:48402, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:26:08,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x34916cd6 to 127.0.0.1:60489 2023-07-22T12:26:08,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:26:08,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:26:08,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure2.ProcedureExecutor(1033): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:26:08,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-07-22T12:26:08,785 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:26:08,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:26:08,791 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:26:08,794 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:26:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:26:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:26:09,206 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:26:09,209 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576}] 2023-07-22T12:26:09,216 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:09,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2023-07-22T12:26:09,377 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:09,379 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2819): Flushing 73a8d9b14b06f35bfa58d9fccd108576 2/2 column families, dataSize=45 B heapSize=632 B 2023-07-22T12:26:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:26:09,453 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/session/764190440e7a41d18df8d98ff5230476 is 49, key is activesession:/session:c/1690028768735/Put/seqid=0 2023-07-22T12:26:09,702 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-07-22T12:26:09,703 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-07-22T12:26:09,872 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/session/764190440e7a41d18df8d98ff5230476 2023-07-22T12:26:09,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:26:09,945 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 764190440e7a41d18df8d98ff5230476 2023-07-22T12:26:09,950 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/session/764190440e7a41d18df8d98ff5230476 as hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/764190440e7a41d18df8d98ff5230476 2023-07-22T12:26:09,960 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/764190440e7a41d18df8d98ff5230476, entries=1, sequenceid=5, filesize=4.9 K 2023-07-22T12:26:09,965 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for 73a8d9b14b06f35bfa58d9fccd108576 in 586ms, sequenceid=5, compaction requested=false 2023-07-22T12:26:09,968 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2520): Flush status journal for 73a8d9b14b06f35bfa58d9fccd108576: 2023-07-22T12:26:09,968 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. for snapshot_backup_system completed. 2023-07-22T12:26:09,969 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:26:09,979 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:26:09,986 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-07-22T12:26:09,986 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/764190440e7a41d18df8d98ff5230476] hfiles 2023-07-22T12:26:09,986 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/764190440e7a41d18df8d98ff5230476 for snapshot=snapshot_backup_system 2023-07-22T12:26:10,404 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:10,408 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=33 2023-07-22T12:26:10,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.HMaster(3994): Remote procedure done, pid=33 2023-07-22T12:26:10,412 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:10,413 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:10,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=33, resume processing ppid=32 2023-07-22T12:26:10,416 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:26:10,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=33, ppid=32, state=SUCCESS; SnapshotRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576 in 1.2050 sec 2023-07-22T12:26:10,418 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:26:10,421 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:26:10,421 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:26:10,424 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:26:10,851 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:26:10,865 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:26:10,866 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:26:10,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:26:11,111 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:26:11,707 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:26:11,707 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-07-22T12:26:11,709 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=32, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.9310 sec 2023-07-22T12:26:12,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:26:12,897 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 32 completed 2023-07-22T12:26:12,898 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] impl.TableBackupClient(115): Backup backup_1690028764280 started at 1690028772897. 2023-07-22T12:26:12,927 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.TableBackupClient(119): Backup session backup_1690028764280 has been started. 2023-07-22T12:26:12,931 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-07-22T12:26:12,947 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-07-22T12:26:12,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-07-22T12:26:12,955 INFO [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-07-22T12:26:12,955 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-07-22T12:26:12,957 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-07-22T12:26:12,957 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-07-22T12:26:12,957 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-07-22T12:26:12,958 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:26:12,959 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45275-0x100005982690001, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-07-22T12:26:12,959 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:12,959 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:26:12,959 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:26:12,960 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:12,960 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-07-22T12:26:12,966 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-07-22T12:26:12,966 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45275-0x100005982690001, quorum=127.0.0.1:60489, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-07-22T12:26:12,967 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-07-22T12:26:12,967 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:26:12,967 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-07-22T12:26:12,969 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-07-22T12:26:12,969 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-07-22T12:26:12,971 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-07-22T12:26:12,971 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-07-22T12:26:12,972 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-07-22T12:26:12,972 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-07-22T12:26:12,972 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,45275,1690028749134' joining acquired barrier for procedure (rolllog) in zk 2023-07-22T12:26:12,975 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-07-22T12:26:12,975 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:12,975 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:12,975 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:26:12,975 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:26:12,975 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45275-0x100005982690001, quorum=127.0.0.1:60489, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-07-22T12:26:12,975 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-07-22T12:26:12,976 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:26:12,976 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:26:12,977 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:26:12,977 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:12,978 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:26:12,978 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,45275,1690028749134' joining acquired barrier for procedure 'rolllog' on coordinator 2023-07-22T12:26:12,979 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-07-22T12:26:12,979 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@73918eb5[Count = 0] remaining members to acquire global barrier 2023-07-22T12:26:12,979 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-07-22T12:26:12,981 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45275-0x100005982690001, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:26:12,981 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-07-22T12:26:12,981 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-07-22T12:26:12,981 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-07-22T12:26:12,981 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:12,981 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-07-22T12:26:12,982 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-07-22T12:26:12,982 DEBUG [rs(jenkins-hbase3.apache.org,45275,1690028749134)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:12,982 INFO [rs(jenkins-hbase3.apache.org,45275,1690028749134)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1690028751553 highest: 1690028751553 on jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:12,983 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45275%2C1690028749134.meta:.meta(num 1690028752029) roll requested 2023-07-22T12:26:12,999 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.meta.1690028772983.meta, exclude list is [], retry=0 2023-07-22T12:26:13,010 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35109,DS-940ef494-c177-4702-ad53-b2959f1d3b03,DISK] 2023-07-22T12:26:13,015 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.meta.1690028752029.meta with entries=36, filesize=11.13 KB; new WAL /user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.meta.1690028772983.meta 2023-07-22T12:26:13,016 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:35109,DS-940ef494-c177-4702-ad53-b2959f1d3b03,DISK]] 2023-07-22T12:26:13,016 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.meta.1690028752029.meta is not closed yet, will try archiving it next time 2023-07-22T12:26:13,016 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45275%2C1690028749134:(num 1690028751553) roll requested 2023-07-22T12:26:13,034 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.1690028773016, exclude list is [], retry=0 2023-07-22T12:26:13,040 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35109,DS-940ef494-c177-4702-ad53-b2959f1d3b03,DISK] 2023-07-22T12:26:13,044 DEBUG [rs(jenkins-hbase3.apache.org,45275,1690028749134)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 62 2023-07-22T12:26:13,044 INFO [rs(jenkins-hbase3.apache.org,45275,1690028749134)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1690028773016 on jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:13,045 DEBUG [rs(jenkins-hbase3.apache.org,45275,1690028749134)-backup-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:26:13,046 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.1690028751553 with entries=19, filesize=3.99 KB; new WAL /user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.1690028773016 2023-07-22T12:26:13,047 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:35109,DS-940ef494-c177-4702-ad53-b2959f1d3b03,DISK]] 2023-07-22T12:26:13,047 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.1690028751553 is not closed yet, will try archiving it next time 2023-07-22T12:26:13,048 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:53634, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-07-22T12:26:13,080 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-07-22T12:26:13,080 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,45275,1690028749134' in zk 2023-07-22T12:26:13,086 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:13,086 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-07-22T12:26:13,086 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:26:13,086 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:13,086 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-07-22T12:26:13,087 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:26:13,087 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:26:13,088 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:26:13,088 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:26:13,089 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:26:13,089 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:13,090 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:26:13,090 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:26:13,091 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:13,092 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,45275,1690028749134': 2023-07-22T12:26:13,092 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,45275,1690028749134' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-07-22T12:26:13,092 INFO [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-07-22T12:26:13,092 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-07-22T12:26:13,092 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-07-22T12:26:13,092 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-07-22T12:26:13,092 INFO [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-07-22T12:26:13,095 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45275-0x100005982690001, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:26:13,095 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:26:13,095 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45275-0x100005982690001, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-07-22T12:26:13,095 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-07-22T12:26:13,095 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-07-22T12:26:13,095 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:26:13,095 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:26:13,095 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:26:13,096 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:26:13,096 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:13,096 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:26:13,097 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:26:13,097 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:26:13,097 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:26:13,097 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:26:13,098 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:26:13,098 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:13,098 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:26:13,098 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:13,099 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:26:13,100 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:13,110 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:13,110 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-07-22T12:26:13,110 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45275-0x100005982690001, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-07-22T12:26:13,110 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45275-0x100005982690001, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-07-22T12:26:13,110 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-07-22T12:26:13,110 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:26:13,110 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:26:13,110 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:13,111 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:26:13,111 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:26:13,111 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:26:13,111 DEBUG [Listener at localhost/44609-EventThread {}] zookeeper.ZKWatcher(604): master:34453-0x100005982690000, quorum=127.0.0.1:60489, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:26:13,111 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:26:13,113 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:26:13,113 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-07-22T12:26:13,113 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-07-22T12:26:13,117 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] client.HBaseAdmin(2751): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 100000 ms per retry) 2023-07-22T12:26:13,117 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] client.HBaseAdmin(2759): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-07-22T12:26:13,217 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] client.HBaseAdmin(2765): Getting current status of procedure from master... 2023-07-22T12:26:13,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-07-22T12:26:13,229 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1690028773228_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } 2023-07-22T12:26:13,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690028773230 (current time:1690028773230). 2023-07-22T12:26:13,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:26:13,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1690028773228_default_test-1690028758503 VERSION not specified, setting to 2 2023-07-22T12:26:13,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x61d09dd8 to 127.0.0.1:60489 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:26:13,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36b9c29f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:26:13,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:26:13,241 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:40156, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:26:13,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x61d09dd8 to 127.0.0.1:60489 2023-07-22T12:26:13,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:26:13,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:26:13,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure2.ProcedureExecutor(1033): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690028773228_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } 2023-07-22T12:26:13,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1690028773228_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-07-22T12:26:13,246 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690028773228_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:26:13,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:26:13,247 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690028773228_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:26:13,250 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690028773228_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:26:13,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:26:13,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:26:13,663 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690028773228_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:26:13,663 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 648e0182e924cbe38699c76e79cee0a5}] 2023-07-22T12:26:13,666 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 648e0182e924cbe38699c76e79cee0a5 2023-07-22T12:26:13,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2023-07-22T12:26:13,819 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5. 2023-07-22T12:26:13,820 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2819): Flushing 648e0182e924cbe38699c76e79cee0a5 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-07-22T12:26:13,842 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/default/test-1690028758503/648e0182e924cbe38699c76e79cee0a5/.tmp/f/3c8f99d0de094dbb9a6f3cb1dadc3824 is 37, key is row10/f:q1/1690028760165/Put/seqid=0 2023-07-22T12:26:13,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:26:13,860 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/default/test-1690028758503/648e0182e924cbe38699c76e79cee0a5/.tmp/f/3c8f99d0de094dbb9a6f3cb1dadc3824 2023-07-22T12:26:13,869 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3c8f99d0de094dbb9a6f3cb1dadc3824 2023-07-22T12:26:13,870 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/default/test-1690028758503/648e0182e924cbe38699c76e79cee0a5/.tmp/f/3c8f99d0de094dbb9a6f3cb1dadc3824 as hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/default/test-1690028758503/648e0182e924cbe38699c76e79cee0a5/f/3c8f99d0de094dbb9a6f3cb1dadc3824 2023-07-22T12:26:13,879 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/default/test-1690028758503/648e0182e924cbe38699c76e79cee0a5/f/3c8f99d0de094dbb9a6f3cb1dadc3824, entries=99, sequenceid=103, filesize=8.2 K 2023-07-22T12:26:13,881 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 648e0182e924cbe38699c76e79cee0a5 in 61ms, sequenceid=103, compaction requested=false 2023-07-22T12:26:13,882 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2520): Flush status journal for 648e0182e924cbe38699c76e79cee0a5: 2023-07-22T12:26:13,882 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5. for snapshot_1690028773228_default_test-1690028758503 completed. 2023-07-22T12:26:13,882 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(240): Storing 'test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5.' region-info for snapshot=snapshot_1690028773228_default_test-1690028758503 2023-07-22T12:26:13,882 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:26:13,882 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/default/test-1690028758503/648e0182e924cbe38699c76e79cee0a5/f/3c8f99d0de094dbb9a6f3cb1dadc3824] hfiles 2023-07-22T12:26:13,882 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/default/test-1690028758503/648e0182e924cbe38699c76e79cee0a5/f/3c8f99d0de094dbb9a6f3cb1dadc3824 for snapshot=snapshot_1690028773228_default_test-1690028758503 2023-07-22T12:26:13,894 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5. 2023-07-22T12:26:13,894 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=35 2023-07-22T12:26:13,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.HMaster(3994): Remote procedure done, pid=35 2023-07-22T12:26:13,895 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1690028773228_default_test-1690028758503 on region 648e0182e924cbe38699c76e79cee0a5 2023-07-22T12:26:13,895 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 648e0182e924cbe38699c76e79cee0a5 2023-07-22T12:26:13,899 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=35, resume processing ppid=34 2023-07-22T12:26:13,899 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=35, ppid=34, state=SUCCESS; SnapshotRegionProcedure 648e0182e924cbe38699c76e79cee0a5 in 233 msec 2023-07-22T12:26:13,899 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690028773228_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:26:13,900 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690028773228_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:26:13,900 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690028773228_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:26:13,900 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1690028773228_default_test-1690028758503 2023-07-22T12:26:13,901 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/.tmp/snapshot_1690028773228_default_test-1690028758503 2023-07-22T12:26:14,323 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690028773228_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:26:14,331 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690028773228_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:26:14,332 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/.tmp/snapshot_1690028773228_default_test-1690028758503 to hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/snapshot_1690028773228_default_test-1690028758503 2023-07-22T12:26:14,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:26:14,766 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690028773228_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:26:14,766 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1690028773228_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-07-22T12:26:14,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=34, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1690028773228_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } in 1.5230 sec 2023-07-22T12:26:15,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:26:15,357 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: default:test-1690028758503, procId: 34 completed 2023-07-22T12:26:15,357 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1690028764280 2023-07-22T12:26:15,357 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-07-22T12:26:15,359 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-07-22T12:26:15,361 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1690028764280_test-1690028758503 2023-07-22T12:26:15,361 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1690028773228_default_test-1690028758503 to hdfs://localhost:35695/backupUT/backup_1690028764280/default/test-1690028758503/ 2023-07-22T12:26:15,371 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-07-22T12:26:15,401 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:35695, inputRoot=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb 2023-07-22T12:26:15,401 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,34453,1690028746140_-727737604_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:35695/backupUT/backup_1690028764280/default/test-1690028758503, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:35695/backupUT/backup_1690028764280/default/test-1690028758503/.hbase-snapshot/.tmp/snapshot_1690028773228_default_test-1690028758503 2023-07-22T12:26:15,401 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:35695, inputRoot=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb, snapshotDir=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/snapshot_1690028773228_default_test-1690028758503. 2023-07-22T12:26:15,413 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/snapshot_1690028773228_default_test-1690028758503 to hdfs://localhost:35695/backupUT/backup_1690028764280/default/test-1690028758503/.hbase-snapshot/.tmp/snapshot_1690028773228_default_test-1690028758503 2023-07-22T12:26:15,848 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:15,848 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:15,849 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:15,849 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:16,300 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-07-22T12:26:16,300 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-07-22T12:26:16,858 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/hadoop-3437559372170621470.jar 2023-07-22T12:26:16,858 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:16,859 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:16,859 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:16,860 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:16,860 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:16,860 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:16,861 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:16,861 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:16,862 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:16,863 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:26:16,863 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:26:16,864 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:26:16,864 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:26:16,865 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:26:16,865 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:26:16,866 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:26:16,866 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:26:16,867 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:26:16,867 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:26:16,868 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:26:16,868 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:26:16,872 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:26:16,872 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:26:16,873 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:26:16,873 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:26:16,873 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:26:16,874 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:26:16,875 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:26:16,932 WARN [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-07-22T12:26:16,956 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1690028773228_default_test-1690028758503' hfile list 2023-07-22T12:26:16,966 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-07-22T12:26:17,175 WARN [Listener at jenkins-hbase3.apache.org/43519 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/mapreduce.cluster.local.dir/1690028777126/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:26:17,175 WARN [Listener at jenkins-hbase3.apache.org/43519 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/mapreduce.cluster.local.dir/1690028777126/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:26:17,354 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-07-22T12:26:17,410 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/default/test-1690028758503/648e0182e924cbe38699c76e79cee0a5/f/3c8f99d0de094dbb9a6f3cb1dadc3824 output=hdfs://localhost:35695/backupUT/backup_1690028764280/default/test-1690028758503/archive/data/default/test-1690028758503/648e0182e924cbe38699c76e79cee0a5/f/3c8f99d0de094dbb9a6f3cb1dadc3824 2023-07-22T12:26:17,410 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 4.004M/sec 2023-07-22T12:26:18,254 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-07-22T12:26:18,255 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-07-22T12:26:18,262 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1690028773228_default_test-1690028758503 2023-07-22T12:26:18,262 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1690028773228_default_test-1690028758503 finished. 2023-07-22T12:26:18,263 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1014): test-1690028758503 2023-07-22T12:26:18,333 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690028764280 2023-07-22T12:26:18,333 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-07-22T12:26:18,350 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:35695/backupUT/backup_1690028764280/.backup.manifest 2023-07-22T12:26:18,350 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.TableBackupClient(358): Backup backup_1690028764280 finished: type=FULL,tablelist=test-1690028758503,targetRootDir=hdfs://localhost:35695/backupUT,startts=1690028772897,completets=1690028778331,bytescopied=0 2023-07-22T12:26:18,350 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-07-22T12:26:18,350 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1690028773228_default_test-1690028758503 2023-07-22T12:26:18,359 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1690028773228_default_test-1690028758503" 2023-07-22T12:26:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1690028773228_default_test-1690028758503 2023-07-22T12:26:18,364 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1690028773228_default_test-1690028758503 for backup backup_1690028764280 succeeded. 2023-07-22T12:26:18,365 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:26:18,377 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:26:18,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:26:18,380 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:26:18,387 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:26:18,404 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] impl.TableBackupClient(376): Backup backup_1690028764280 completed. 2023-07-22T12:26:18,404 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] client.ConnectionImplementation(2095): Closing master protocol: MasterService 2023-07-22T12:26:18,404 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4c0443b7 to 127.0.0.1:60489 2023-07-22T12:26:18,404 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:26:18,430 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] backup.TestBackupDeleteWithFailures(147): backup complete 2023-07-22T12:26:18,454 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:26:18,463 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:26:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690028778463 (current time:1690028778463). 2023-07-22T12:26:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:26:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:26:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x63038f82 to 127.0.0.1:60489 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:26:18,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ce743d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:26:18,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:26:18,472 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:40160, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:26:18,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x63038f82 to 127.0.0.1:60489 2023-07-22T12:26:18,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:26:18,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:26:18,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure2.ProcedureExecutor(1033): Stored pid=36, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:26:18,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-07-22T12:26:18,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-07-22T12:26:18,476 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:26:18,477 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:26:18,479 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:26:18,487 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:26:18,487 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576}] 2023-07-22T12:26:18,489 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-07-22T12:26:18,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=37 2023-07-22T12:26:18,641 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:18,642 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2819): Flushing 73a8d9b14b06f35bfa58d9fccd108576 2/2 column families, dataSize=1.14 KB heapSize=2.37 KB 2023-07-22T12:26:18,657 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/meta/214bafad8f074d54a243bd148648f671 is 169, key is trslm:hdfs://localhost:35695/backupUT\x00test-1690028758503/meta:log-roll-map/1690028778321/Put/seqid=0 2023-07-22T12:26:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-07-22T12:26:19,062 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=586 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/meta/214bafad8f074d54a243bd148648f671 2023-07-22T12:26:19,069 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 214bafad8f074d54a243bd148648f671 2023-07-22T12:26:19,072 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/session/ff0d1b4b15d542d786093c22d34f0ff7 is 310, key is session:backup_1690028764280/session:context/1690028778386/Put/seqid=0 2023-07-22T12:26:19,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-07-22T12:26:19,477 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=579 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/session/ff0d1b4b15d542d786093c22d34f0ff7 2023-07-22T12:26:19,485 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: ff0d1b4b15d542d786093c22d34f0ff7 2023-07-22T12:26:19,486 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/meta/214bafad8f074d54a243bd148648f671 as hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/214bafad8f074d54a243bd148648f671 2023-07-22T12:26:19,493 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/214bafad8f074d54a243bd148648f671, entries=5, sequenceid=18, filesize=5.5 K 2023-07-22T12:26:19,494 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/session/ff0d1b4b15d542d786093c22d34f0ff7 as hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/ff0d1b4b15d542d786093c22d34f0ff7 2023-07-22T12:26:19,500 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/ff0d1b4b15d542d786093c22d34f0ff7, entries=2, sequenceid=18, filesize=5.3 K 2023-07-22T12:26:19,501 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1165, heapSize ~2.34 KB/2392, currentSize=0 B/0 for 73a8d9b14b06f35bfa58d9fccd108576 in 860ms, sequenceid=18, compaction requested=false 2023-07-22T12:26:19,501 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2520): Flush status journal for 73a8d9b14b06f35bfa58d9fccd108576: 2023-07-22T12:26:19,501 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. for snapshot_backup_system completed. 2023-07-22T12:26:19,502 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:26:19,502 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:26:19,502 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/214bafad8f074d54a243bd148648f671] hfiles 2023-07-22T12:26:19,502 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/214bafad8f074d54a243bd148648f671 for snapshot=snapshot_backup_system 2023-07-22T12:26:19,502 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/764190440e7a41d18df8d98ff5230476, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/ff0d1b4b15d542d786093c22d34f0ff7] hfiles 2023-07-22T12:26:19,502 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/764190440e7a41d18df8d98ff5230476 for snapshot=snapshot_backup_system 2023-07-22T12:26:19,502 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/ff0d1b4b15d542d786093c22d34f0ff7 for snapshot=snapshot_backup_system 2023-07-22T12:26:19,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-07-22T12:26:19,908 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:19,908 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=37 2023-07-22T12:26:19,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.HMaster(3994): Remote procedure done, pid=37 2023-07-22T12:26:19,909 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:19,910 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:19,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=37, resume processing ppid=36 2023-07-22T12:26:19,913 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:26:19,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=37, ppid=36, state=SUCCESS; SnapshotRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576 in 1.4240 sec 2023-07-22T12:26:19,914 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:26:19,914 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:26:19,914 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:26:19,915 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:26:20,327 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:26:20,336 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:26:20,337 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:26:20,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-07-22T12:26:20,755 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:26:20,755 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-07-22T12:26:20,757 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=36, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2820 sec 2023-07-22T12:26:21,042 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-07-22T12:26:21,044 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:54156, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-07-22T12:26:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-07-22T12:26:22,585 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 36 completed 2023-07-22T12:26:22,592 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupAdminImpl(227): Deleting backup backup_1690028764280 ... 2023-07-22T12:26:22,593 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] util.BackupUtils(435): Delete log files: .backup.manifest 2023-07-22T12:26:22,594 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] util.BackupUtils(435): Delete log files: default 2023-07-22T12:26:22,595 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] util.BackupUtils(443): Trying to cleanup up target dir : backup_1690028764280 2023-07-22T12:26:22,595 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] util.BackupUtils(458): No data has been found in hdfs://localhost:35695/backupUT/backup_1690028764280/default/test-1690028758503. 2023-07-22T12:26:22,599 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] util.BackupUtils(465): hdfs://localhost:35695/backupUT/backup_1690028764280/default is empty, remove it. 2023-07-22T12:26:22,607 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupAdminImpl(269): 0 bulk loaded files out of 0 were deleted 2023-07-22T12:26:22,607 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(433): deleted 0 original bulkload rows 2023-07-22T12:26:22,610 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupAdminImpl(276): Delete backup backup_1690028764280 completed. 2023-07-22T12:26:22,615 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1014): test-1690028758503 2023-07-22T12:26:22,618 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:26:22,623 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:26:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.MetricsHBaseServer(144): Unknown exception type java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1155) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:375) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:760) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-07-22T12:26:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.CallRunner(138): callId: 143 service: MasterService methodName: DeleteSnapshot size: 57 connection: 172.31.12.81:46424 deadline: 1690028842623, exception=java.io.IOException: preDeleteSnapshot 2023-07-22T12:26:22,729 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:26:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.MetricsHBaseServer(144): Unknown exception type java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1155) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:375) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:760) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-07-22T12:26:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.CallRunner(138): callId: 145 service: MasterService methodName: DeleteSnapshot size: 57 connection: 172.31.12.81:46424 deadline: 1690028842728, exception=java.io.IOException: preDeleteSnapshot 2023-07-22T12:26:22,737 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-07-22T12:26:22,743 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] client.HBaseAdmin$18(963): Started disable of backup:system 2023-07-22T12:26:22,748 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable backup:system 2023-07-22T12:26:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure2.ProcedureExecutor(1033): Stored pid=38, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-07-22T12:26:22,760 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028782760"}]},"ts":"1690028782760"} 2023-07-22T12:26:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-07-22T12:26:22,762 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-07-22T12:26:22,763 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-07-22T12:26:22,765 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=73a8d9b14b06f35bfa58d9fccd108576, UNASSIGN}] 2023-07-22T12:26:22,766 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=73a8d9b14b06f35bfa58d9fccd108576, UNASSIGN 2023-07-22T12:26:22,767 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=39 updating hbase:meta row=73a8d9b14b06f35bfa58d9fccd108576, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:22,769 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE; CloseRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576, server=jenkins-hbase3.apache.org,45275,1690028749134}] 2023-07-22T12:26:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-07-22T12:26:22,929 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(119): Close 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:22,929 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1668): Closing 73a8d9b14b06f35bfa58d9fccd108576, disabling compactions & flushes 2023-07-22T12:26:22,929 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1690): Closing region backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:22,929 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:22,929 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. after waiting 0 ms 2023-07-22T12:26:22,929 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:22,929 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(2819): Flushing 73a8d9b14b06f35bfa58d9fccd108576 2/2 column families, dataSize=179 B heapSize=912 B 2023-07-22T12:26:22,934 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/meta/e8cc1d20fb094f09aae46a6fbb826647 is 91, key is incrbackupset:hdfs://localhost:35695/backupUT/meta:test-1690028758503/1690028782616/Put/seqid=0 2023-07-22T12:26:22,938 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=124 B at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/meta/e8cc1d20fb094f09aae46a6fbb826647 2023-07-22T12:26:22,945 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e8cc1d20fb094f09aae46a6fbb826647 2023-07-22T12:26:22,945 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e8cc1d20fb094f09aae46a6fbb826647 2023-07-22T12:26:22,950 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/session/e75f20ffc5c0493b9ca7a156dbaa5498 is 59, key is session:backup_1690028764280/session:/1690028782608/DeleteFamily/seqid=0 2023-07-22T12:26:23,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-07-22T12:26:23,355 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55 B at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/session/e75f20ffc5c0493b9ca7a156dbaa5498 2023-07-22T12:26:23,364 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e75f20ffc5c0493b9ca7a156dbaa5498 2023-07-22T12:26:23,364 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e75f20ffc5c0493b9ca7a156dbaa5498 2023-07-22T12:26:23,366 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/meta/e8cc1d20fb094f09aae46a6fbb826647 as hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/e8cc1d20fb094f09aae46a6fbb826647 2023-07-22T12:26:23,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-07-22T12:26:23,373 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e8cc1d20fb094f09aae46a6fbb826647 2023-07-22T12:26:23,373 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/e8cc1d20fb094f09aae46a6fbb826647, entries=2, sequenceid=24, filesize=5.2 K 2023-07-22T12:26:23,375 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/session/e75f20ffc5c0493b9ca7a156dbaa5498 as hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/e75f20ffc5c0493b9ca7a156dbaa5498 2023-07-22T12:26:23,383 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e75f20ffc5c0493b9ca7a156dbaa5498 2023-07-22T12:26:23,383 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/e75f20ffc5c0493b9ca7a156dbaa5498, entries=1, sequenceid=24, filesize=5.1 K 2023-07-22T12:26:23,384 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(3022): Finished flush of dataSize ~179 B/179, heapSize ~880 B/880, currentSize=0 B/0 for 73a8d9b14b06f35bfa58d9fccd108576 in 455ms, sequenceid=24, compaction requested=true 2023-07-22T12:26:23,386 DEBUG [StoreFileCloser-73a8d9b14b06f35bfa58d9fccd108576-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 214bafad8f074d54a243bd148648f671 2023-07-22T12:26:23,388 DEBUG [StoreFileCloser-73a8d9b14b06f35bfa58d9fccd108576-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e8cc1d20fb094f09aae46a6fbb826647 2023-07-22T12:26:23,390 DEBUG [StoreFileCloser-73a8d9b14b06f35bfa58d9fccd108576-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 764190440e7a41d18df8d98ff5230476 2023-07-22T12:26:23,391 DEBUG [StoreFileCloser-73a8d9b14b06f35bfa58d9fccd108576-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: ff0d1b4b15d542d786093c22d34f0ff7 2023-07-22T12:26:23,393 DEBUG [StoreFileCloser-73a8d9b14b06f35bfa58d9fccd108576-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e75f20ffc5c0493b9ca7a156dbaa5498 2023-07-22T12:26:23,398 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/recovered.edits/27.seqid, newMaxSeqId=27, maxSeqId=1 2023-07-22T12:26:23,399 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:26:23,399 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1909): Closed backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:23,400 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1622): Region close journal for 73a8d9b14b06f35bfa58d9fccd108576: 2023-07-22T12:26:23,402 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(163): Closed 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:23,402 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=39 updating hbase:meta row=73a8d9b14b06f35bfa58d9fccd108576, regionState=CLOSED 2023-07-22T12:26:23,406 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=40, resume processing ppid=39 2023-07-22T12:26:23,406 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=40, ppid=39, state=SUCCESS; CloseRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576, server=jenkins-hbase3.apache.org,45275,1690028749134 in 635 msec 2023-07-22T12:26:23,408 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=39, resume processing ppid=38 2023-07-22T12:26:23,408 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=39, ppid=38, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=73a8d9b14b06f35bfa58d9fccd108576, UNASSIGN in 642 msec 2023-07-22T12:26:23,409 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028783408"}]},"ts":"1690028783408"} 2023-07-22T12:26:23,410 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-07-22T12:26:23,413 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-07-22T12:26:23,415 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=38, state=SUCCESS; DisableTableProcedure table=backup:system in 663 msec 2023-07-22T12:26:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-07-22T12:26:23,868 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] client.HBaseAdmin$TableFuture(3590): Operation: DISABLE, Table Name: backup:system, procId: 38 completed 2023-07-22T12:26:23,878 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] client.HBaseAdmin(2645): Taking restore-failsafe snapshot: hbase-failsafe-snapshot_backup_system-1690028783878 2023-07-22T12:26:23,879 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1690028783878 table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:26:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690028783879 (current time:1690028783879). 2023-07-22T12:26:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:26:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1690028783878 VERSION not specified, setting to 2 2023-07-22T12:26:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3b9ff6e8 to 127.0.0.1:60489 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:26:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1998f170, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:26:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:26:23,887 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:40068, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:26:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3b9ff6e8 to 127.0.0.1:60489 2023-07-22T12:26:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:26:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:26:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure2.ProcedureExecutor(1033): Stored pid=41, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028783878 table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:26:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028783878 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 41 2023-07-22T12:26:23,891 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028783878 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:26:23,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-07-22T12:26:23,892 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028783878 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:26:23,894 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028783878 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:26:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-07-22T12:26:24,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-07-22T12:26:24,300 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028783878 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-07-22T12:26:24,302 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-07-22T12:26:24,302 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-07-22T12:26:24,302 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:26:24,304 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/214bafad8f074d54a243bd148648f671, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/e8cc1d20fb094f09aae46a6fbb826647] hfiles 2023-07-22T12:26:24,304 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/2): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/214bafad8f074d54a243bd148648f671 2023-07-22T12:26:24,304 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/2): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/e8cc1d20fb094f09aae46a6fbb826647 2023-07-22T12:26:24,304 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:26:24,306 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/764190440e7a41d18df8d98ff5230476, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/e75f20ffc5c0493b9ca7a156dbaa5498, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/ff0d1b4b15d542d786093c22d34f0ff7] hfiles 2023-07-22T12:26:24,306 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/3): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/764190440e7a41d18df8d98ff5230476 2023-07-22T12:26:24,306 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/3): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/e75f20ffc5c0493b9ca7a156dbaa5498 2023-07-22T12:26:24,306 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/3): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/ff0d1b4b15d542d786093c22d34f0ff7 2023-07-22T12:26:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-07-22T12:26:24,712 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 73a8d9b14b06f35bfa58d9fccd108576, NAME => 'backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-07-22T12:26:24,713 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028783878 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:26:24,714 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028783878 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:26:24,714 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1690028783878 2023-07-22T12:26:24,715 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1690028783878 2023-07-22T12:26:24,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-07-22T12:26:25,126 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028783878 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:26:25,134 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028783878 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:26:25,135 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1690028783878 to hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1690028783878 2023-07-22T12:26:25,956 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028783878 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:26:25,956 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028783878 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 41 2023-07-22T12:26:25,958 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=41, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028783878 table=backup:system type=FLUSH ttl=0 } in 2.0680 sec 2023-07-22T12:26:25,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-07-22T12:26:25,998 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 41 completed 2023-07-22T12:26:26,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure2.ProcedureExecutor(1033): Stored pid=42, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690028778463 type: FLUSH version: 2 ttl: 0 ) 2023-07-22T12:26:26,025 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(1015): Restore snapshot=snapshot_backup_system as table=backup:system 2023-07-22T12:26:26,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-07-22T12:26:26,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-07-22T12:26:26,300 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-07-22T12:26:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-07-22T12:26:26,440 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/.tabledesc/.tableinfo.0000000001.635 2023-07-22T12:26:26,441 INFO [PEWorker-2 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2023-07-22T12:26:26,442 INFO [PEWorker-2 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:26:26,448 INFO [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690028778463 type: FLUSH version: 2 ttl: 0 2023-07-22T12:26:26,448 DEBUG [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system 2023-07-22T12:26:26,451 DEBUG [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-07-22T12:26:26,451 INFO [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(227): region to restore: 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:26,465 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/e8cc1d20fb094f09aae46a6fbb826647 to hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/archive/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/e8cc1d20fb094f09aae46a6fbb826647 2023-07-22T12:26:26,466 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:26:26,469 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/e75f20ffc5c0493b9ca7a156dbaa5498 to hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/archive/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/e75f20ffc5c0493b9ca7a156dbaa5498 2023-07-22T12:26:26,469 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:26:26,469 INFO [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690028778463 type: FLUSH version: 2 ttl: 0 2023-07-22T12:26:26,471 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1690028786470"}]},"ts":"9223372036854775807"} 2023-07-22T12:26:26,473 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-07-22T12:26:26,473 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 73a8d9b14b06f35bfa58d9fccd108576, NAME => 'backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:26:26,473 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1690028786471"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690028786471"}]},"ts":"1690028786471"} 2023-07-22T12:26:26,475 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:26:26,475 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-07-22T12:26:26,475 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => 73a8d9b14b06f35bfa58d9fccd108576, NAME => 'backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:26:26,475 INFO [PEWorker-2 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-07-22T12:26:26,478 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=42, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690028778463 type: FLUSH version: 2 ttl: 0 ) in 462 msec 2023-07-22T12:26:26,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-07-22T12:26:26,633 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] client.HBaseAdmin$TableFuture(3590): Operation: MODIFY, Table Name: backup:system, procId: 42 completed 2023-07-22T12:26:26,634 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] client.HBaseAdmin(2678): Deleting restore-failsafe snapshot: hbase-failsafe-snapshot_backup_system-1690028783878 2023-07-22T12:26:26,634 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1690028783878" 2023-07-22T12:26:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.MetricsHBaseServer(144): Unknown exception type java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1155) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:375) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:760) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-07-22T12:26:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.CallRunner(138): callId: 192 service: MasterService methodName: DeleteSnapshot size: 86 connection: 172.31.12.81:46424 deadline: 1690028846634, exception=java.io.IOException: preDeleteSnapshot 2023-07-22T12:26:26,740 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1690028783878" 2023-07-22T12:26:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.MetricsHBaseServer(144): Unknown exception type java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1155) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:375) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:760) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-07-22T12:26:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.CallRunner(138): callId: 194 service: MasterService methodName: DeleteSnapshot size: 86 connection: 172.31.12.81:46424 deadline: 1690028846739, exception=java.io.IOException: preDeleteSnapshot 2023-07-22T12:26:26,744 ERROR [Listener at jenkins-hbase3.apache.org/43519 {}] client.HBaseAdmin(2681): Unable to remove the failsafe snapshot: hbase-failsafe-snapshot_backup_system-1690028783878 org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2023-07-22T12:26:26.638Z, RpcRetryingCaller{globalStartTime=2023-07-22T12:26:26.634Z, pause=100, maxAttempts=2}, java.io.IOException: java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1155) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1152) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1152) at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:375) at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:760) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) 2023-07-22T12:26:26.744Z, RpcRetryingCaller{globalStartTime=2023-07-22T12:26:26.634Z, pause=100, maxAttempts=2}, java.io.IOException: java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1155) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1152) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1152) at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:375) at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:760) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:142) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3018) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3010) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.deleteSnapshot(HBaseAdmin.java:2918) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.restoreSnapshot(HBaseAdmin.java:2679) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.restoreSnapshot(HBaseAdmin.java:2617) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.restoreSnapshot(HBaseAdmin.java:2582) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.backup.impl.BackupSystemTable.restoreFromSnapshot(BackupSystemTable.java:1523) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.deleteBackups(BackupAdminImpl.java:155) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures.testBackupDeleteWithFailuresAfter(TestBackupDeleteWithFailures.java:161) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures.testBackupDeleteWithFailures(TestBackupDeleteWithFailures.java:136) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1155) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1152) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1152) at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:375) at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:760) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.MasterCallable.call(MasterCallable.java:101) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] ... 34 more Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1155) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1152) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1152) at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:375) at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:760) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:388) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:92) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:420) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:114) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:129) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:165) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:195) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] ... 1 more 2023-07-22T12:26:26,748 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] client.HBaseAdmin$17(903): Started enable of backup:system 2023-07-22T12:26:26,754 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.HMaster$11(2630): Client=jenkins//172.31.12.81 enable backup:system 2023-07-22T12:26:26,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure2.ProcedureExecutor(1033): Stored pid=43, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-07-22T12:26:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=43 2023-07-22T12:26:26,765 INFO [PEWorker-3 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-07-22T12:26:26,766 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028786765"}]},"ts":"1690028786765"} 2023-07-22T12:26:26,767 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-07-22T12:26:26,770 DEBUG [PEWorker-3 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-07-22T12:26:26,770 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=73a8d9b14b06f35bfa58d9fccd108576, ASSIGN}] 2023-07-22T12:26:26,772 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=73a8d9b14b06f35bfa58d9fccd108576, ASSIGN 2023-07-22T12:26:26,773 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=73a8d9b14b06f35bfa58d9fccd108576, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-07-22T12:26:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=43 2023-07-22T12:26:26,925 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=44 updating hbase:meta row=73a8d9b14b06f35bfa58d9fccd108576, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:26,927 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE; OpenRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576, server=jenkins-hbase3.apache.org,45275,1690028749134}] 2023-07-22T12:26:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=43 2023-07-22T12:26:27,083 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(130): Open backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:27,083 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7254): Opening region: {ENCODED => 73a8d9b14b06f35bfa58d9fccd108576, NAME => 'backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:26:27,083 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:26:27,083 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:27,083 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(888): Instantiated backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:26:27,083 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7296): checking encryption for 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:27,084 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7299): checking classloading for 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:27,085 INFO [StoreOpener-73a8d9b14b06f35bfa58d9fccd108576-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:27,086 INFO [StoreOpener-73a8d9b14b06f35bfa58d9fccd108576-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73a8d9b14b06f35bfa58d9fccd108576 columnFamilyName meta 2023-07-22T12:26:27,086 DEBUG [StoreOpener-73a8d9b14b06f35bfa58d9fccd108576-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:26:27,093 DEBUG [StoreOpener-73a8d9b14b06f35bfa58d9fccd108576-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/214bafad8f074d54a243bd148648f671 2023-07-22T12:26:27,093 INFO [StoreOpener-73a8d9b14b06f35bfa58d9fccd108576-1 {}] regionserver.HStore(324): Store=73a8d9b14b06f35bfa58d9fccd108576/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:26:27,093 INFO [StoreOpener-73a8d9b14b06f35bfa58d9fccd108576-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:27,094 INFO [StoreOpener-73a8d9b14b06f35bfa58d9fccd108576-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73a8d9b14b06f35bfa58d9fccd108576 columnFamilyName session 2023-07-22T12:26:27,094 DEBUG [StoreOpener-73a8d9b14b06f35bfa58d9fccd108576-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:26:27,101 DEBUG [StoreOpener-73a8d9b14b06f35bfa58d9fccd108576-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/764190440e7a41d18df8d98ff5230476 2023-07-22T12:26:27,106 DEBUG [StoreOpener-73a8d9b14b06f35bfa58d9fccd108576-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/ff0d1b4b15d542d786093c22d34f0ff7 2023-07-22T12:26:27,106 INFO [StoreOpener-73a8d9b14b06f35bfa58d9fccd108576-1 {}] regionserver.HStore(324): Store=73a8d9b14b06f35bfa58d9fccd108576/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:26:27,107 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:27,108 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:27,109 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-07-22T12:26:27,111 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1076): writing seq id for 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:27,112 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): Opened 73a8d9b14b06f35bfa58d9fccd108576; next sequenceid=28; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9913623840, jitterRate=-0.07672183215618134}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-07-22T12:26:27,112 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(992): Region open journal for 73a8d9b14b06f35bfa58d9fccd108576: 2023-07-22T12:26:27,113 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576., pid=45, masterSystemTime=1690028787079 2023-07-22T12:26:27,115 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:27,115 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(158): Opened backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:27,115 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=44 updating hbase:meta row=73a8d9b14b06f35bfa58d9fccd108576, regionState=OPEN, openSeqNum=28, regionLocation=jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:27,119 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=45, resume processing ppid=44 2023-07-22T12:26:27,119 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=45, ppid=44, state=SUCCESS; OpenRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576, server=jenkins-hbase3.apache.org,45275,1690028749134 in 190 msec 2023-07-22T12:26:27,121 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=44, resume processing ppid=43 2023-07-22T12:26:27,121 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=44, ppid=43, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=73a8d9b14b06f35bfa58d9fccd108576, ASSIGN in 349 msec 2023-07-22T12:26:27,122 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028787121"}]},"ts":"1690028787121"} 2023-07-22T12:26:27,123 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-07-22T12:26:27,125 INFO [PEWorker-4 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-07-22T12:26:27,127 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=43, state=SUCCESS; EnableTableProcedure table=backup:system in 370 msec 2023-07-22T12:26:27,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=43 2023-07-22T12:26:27,370 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] client.HBaseAdmin$TableFuture(3590): Operation: ENABLE, Table Name: backup:system, procId: 43 completed 2023-07-22T12:26:27,370 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1525): Done restoring backup system table 2023-07-22T12:26:27,370 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:26:27,379 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:26:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.MetricsHBaseServer(144): Unknown exception type java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1155) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:375) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:760) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-07-22T12:26:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.CallRunner(138): callId: 208 service: MasterService methodName: DeleteSnapshot size: 57 connection: 172.31.12.81:46424 deadline: 1690028847379, exception=java.io.IOException: preDeleteSnapshot 2023-07-22T12:26:27,483 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:26:27,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.MetricsHBaseServer(144): Unknown exception type java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1155) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:375) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:760) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-07-22T12:26:27,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.CallRunner(138): callId: 210 service: MasterService methodName: DeleteSnapshot size: 57 connection: 172.31.12.81:46424 deadline: 1690028847483, exception=java.io.IOException: preDeleteSnapshot 2023-07-22T12:26:27,488 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:26:27,522 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cedcd9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:26:27,528 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62884a9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:26:27,529 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:26:27,530 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:54170, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:26:27,534 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:26:27,535 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:40076, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService REPAIR status: no failed sessions found. Checking failed delete backup operation ... Found failed DELETE operation for: backup_1690028764280 Running DELETE again ... 2023-07-22T12:26:27,553 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-07-22T12:26:27,561 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable backup:system 2023-07-22T12:26:27,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure2.ProcedureExecutor(1033): Stored pid=46, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-07-22T12:26:27,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-07-22T12:26:27,567 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028787567"}]},"ts":"1690028787567"} 2023-07-22T12:26:27,569 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-07-22T12:26:27,570 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-07-22T12:26:27,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=73a8d9b14b06f35bfa58d9fccd108576, UNASSIGN}] 2023-07-22T12:26:27,573 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=47, ppid=46, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=73a8d9b14b06f35bfa58d9fccd108576, UNASSIGN 2023-07-22T12:26:27,574 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=47 updating hbase:meta row=73a8d9b14b06f35bfa58d9fccd108576, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:27,575 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE; CloseRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576, server=jenkins-hbase3.apache.org,45275,1690028749134}] 2023-07-22T12:26:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-07-22T12:26:27,727 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] handler.UnassignRegionHandler(119): Close 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:27,728 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1668): Closing 73a8d9b14b06f35bfa58d9fccd108576, disabling compactions & flushes 2023-07-22T12:26:27,728 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1690): Closing region backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:27,728 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:27,728 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. after waiting 0 ms 2023-07-22T12:26:27,728 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:27,728 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(2819): Flushing 73a8d9b14b06f35bfa58d9fccd108576 2/2 column families, dataSize=44 B heapSize=632 B 2023-07-22T12:26:27,743 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/session/5cf993543b0147628a8c663647092a97 is 48, key is activesession:/session:c/1690028787489/Put/seqid=0 2023-07-22T12:26:27,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-07-22T12:26:28,150 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44 B at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/session/5cf993543b0147628a8c663647092a97 2023-07-22T12:26:28,157 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 5cf993543b0147628a8c663647092a97 2023-07-22T12:26:28,158 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/session/5cf993543b0147628a8c663647092a97 as hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/5cf993543b0147628a8c663647092a97 2023-07-22T12:26:28,164 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/5cf993543b0147628a8c663647092a97, entries=1, sequenceid=31, filesize=4.9 K 2023-07-22T12:26:28,165 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(3022): Finished flush of dataSize ~44 B/44, heapSize ~360 B/360, currentSize=0 B/0 for 73a8d9b14b06f35bfa58d9fccd108576 in 437ms, sequenceid=31, compaction requested=true 2023-07-22T12:26:28,167 DEBUG [StoreFileCloser-73a8d9b14b06f35bfa58d9fccd108576-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 214bafad8f074d54a243bd148648f671 2023-07-22T12:26:28,169 DEBUG [StoreFileCloser-73a8d9b14b06f35bfa58d9fccd108576-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 764190440e7a41d18df8d98ff5230476 2023-07-22T12:26:28,170 DEBUG [StoreFileCloser-73a8d9b14b06f35bfa58d9fccd108576-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: ff0d1b4b15d542d786093c22d34f0ff7 2023-07-22T12:26:28,171 DEBUG [StoreFileCloser-73a8d9b14b06f35bfa58d9fccd108576-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 5cf993543b0147628a8c663647092a97 2023-07-22T12:26:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-07-22T12:26:28,176 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/recovered.edits/34.seqid, newMaxSeqId=34, maxSeqId=27 2023-07-22T12:26:28,177 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:26:28,177 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1909): Closed backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:28,177 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1622): Region close journal for 73a8d9b14b06f35bfa58d9fccd108576: 2023-07-22T12:26:28,179 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] handler.UnassignRegionHandler(163): Closed 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:28,179 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=47 updating hbase:meta row=73a8d9b14b06f35bfa58d9fccd108576, regionState=CLOSED 2023-07-22T12:26:28,183 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=48, resume processing ppid=47 2023-07-22T12:26:28,183 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=48, ppid=47, state=SUCCESS; CloseRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576, server=jenkins-hbase3.apache.org,45275,1690028749134 in 606 msec 2023-07-22T12:26:28,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=47, resume processing ppid=46 2023-07-22T12:26:28,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=47, ppid=46, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=73a8d9b14b06f35bfa58d9fccd108576, UNASSIGN in 612 msec 2023-07-22T12:26:28,185 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028788185"}]},"ts":"1690028788185"} 2023-07-22T12:26:28,186 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-07-22T12:26:28,188 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-07-22T12:26:28,190 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=46, state=SUCCESS; DisableTableProcedure table=backup:system in 628 msec 2023-07-22T12:26:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-07-22T12:26:28,688 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1690028788687 table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:26:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690028788688 (current time:1690028788688). 2023-07-22T12:26:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:26:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1690028788687 VERSION not specified, setting to 2 2023-07-22T12:26:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f451a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:26:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:26:28,696 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:40078, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:26:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:26:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:26:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure2.ProcedureExecutor(1033): Stored pid=49, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028788687 table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:26:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028788687 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 49 2023-07-22T12:26:28,700 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028788687 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:26:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-07-22T12:26:28,701 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028788687 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:26:28,703 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028788687 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:26:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-07-22T12:26:28,944 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:26:29,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-07-22T12:26:29,109 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028788687 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-07-22T12:26:29,111 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-07-22T12:26:29,111 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-07-22T12:26:29,111 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:26:29,112 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/214bafad8f074d54a243bd148648f671] hfiles 2023-07-22T12:26:29,112 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/1): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/214bafad8f074d54a243bd148648f671 2023-07-22T12:26:29,112 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:26:29,114 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/5cf993543b0147628a8c663647092a97, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/764190440e7a41d18df8d98ff5230476, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/ff0d1b4b15d542d786093c22d34f0ff7] hfiles 2023-07-22T12:26:29,114 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/3): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/5cf993543b0147628a8c663647092a97 2023-07-22T12:26:29,114 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/3): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/764190440e7a41d18df8d98ff5230476 2023-07-22T12:26:29,114 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/3): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/ff0d1b4b15d542d786093c22d34f0ff7 2023-07-22T12:26:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-07-22T12:26:29,520 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 73a8d9b14b06f35bfa58d9fccd108576, NAME => 'backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-07-22T12:26:29,521 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028788687 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:26:29,522 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028788687 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:26:29,522 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1690028788687 2023-07-22T12:26:29,522 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1690028788687 2023-07-22T12:26:29,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-07-22T12:26:29,932 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028788687 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:26:29,940 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028788687 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:26:29,941 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1690028788687 to hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1690028788687 2023-07-22T12:26:30,759 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028788687 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:26:30,759 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028788687 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 49 2023-07-22T12:26:30,761 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=49, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028788687 table=backup:system type=FLUSH ttl=0 } in 2.0610 sec 2023-07-22T12:26:30,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-07-22T12:26:30,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure2.ProcedureExecutor(1033): Stored pid=50, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690028778463 type: FLUSH version: 2 ttl: 0 ) 2023-07-22T12:26:30,816 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(1015): Restore snapshot=snapshot_backup_system as table=backup:system 2023-07-22T12:26:30,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-07-22T12:26:30,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-07-22T12:26:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-07-22T12:26:31,230 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2023-07-22T12:26:31,230 INFO [PEWorker-2 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/.tabledesc/.tableinfo.0000000003.635 2023-07-22T12:26:31,232 INFO [PEWorker-2 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:26:31,235 INFO [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690028778463 type: FLUSH version: 2 ttl: 0 2023-07-22T12:26:31,235 DEBUG [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system 2023-07-22T12:26:31,238 DEBUG [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-07-22T12:26:31,239 INFO [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(227): region to restore: 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:31,244 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:26:31,248 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/5cf993543b0147628a8c663647092a97 to hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/archive/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/5cf993543b0147628a8c663647092a97 2023-07-22T12:26:31,249 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:26:31,249 INFO [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690028778463 type: FLUSH version: 2 ttl: 0 2023-07-22T12:26:31,251 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1690028791250"}]},"ts":"9223372036854775807"} 2023-07-22T12:26:31,253 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-07-22T12:26:31,253 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 73a8d9b14b06f35bfa58d9fccd108576, NAME => 'backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:26:31,253 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1690028791251"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690028791251"}]},"ts":"1690028791251"} 2023-07-22T12:26:31,255 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:26:31,255 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-07-22T12:26:31,255 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => 73a8d9b14b06f35bfa58d9fccd108576, NAME => 'backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:26:31,255 INFO [PEWorker-2 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-07-22T12:26:31,258 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=50, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690028778463 type: FLUSH version: 2 ttl: 0 ) in 441 msec 2023-07-22T12:26:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-07-22T12:26:31,423 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1690028788687" 2023-07-22T12:26:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(378): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1690028788687 2023-07-22T12:26:31,427 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.HMaster$11(2630): Client=jenkins//172.31.12.81 enable backup:system 2023-07-22T12:26:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure2.ProcedureExecutor(1033): Stored pid=51, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-07-22T12:26:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=51 2023-07-22T12:26:31,431 INFO [PEWorker-3 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-07-22T12:26:31,431 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028791431"}]},"ts":"1690028791431"} 2023-07-22T12:26:31,433 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-07-22T12:26:31,435 DEBUG [PEWorker-3 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-07-22T12:26:31,435 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=73a8d9b14b06f35bfa58d9fccd108576, ASSIGN}] 2023-07-22T12:26:31,437 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=73a8d9b14b06f35bfa58d9fccd108576, ASSIGN 2023-07-22T12:26:31,437 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=73a8d9b14b06f35bfa58d9fccd108576, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-07-22T12:26:31,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=51 2023-07-22T12:26:31,589 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=73a8d9b14b06f35bfa58d9fccd108576, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:31,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; OpenRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576, server=jenkins-hbase3.apache.org,45275,1690028749134}] 2023-07-22T12:26:31,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=51 2023-07-22T12:26:31,745 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(130): Open backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:31,746 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7254): Opening region: {ENCODED => 73a8d9b14b06f35bfa58d9fccd108576, NAME => 'backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:26:31,746 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:26:31,746 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:31,746 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(888): Instantiated backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:26:31,746 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7296): checking encryption for 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:31,746 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7299): checking classloading for 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:31,748 INFO [StoreOpener-73a8d9b14b06f35bfa58d9fccd108576-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:31,749 INFO [StoreOpener-73a8d9b14b06f35bfa58d9fccd108576-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73a8d9b14b06f35bfa58d9fccd108576 columnFamilyName meta 2023-07-22T12:26:31,749 DEBUG [StoreOpener-73a8d9b14b06f35bfa58d9fccd108576-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:26:31,755 DEBUG [StoreOpener-73a8d9b14b06f35bfa58d9fccd108576-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/214bafad8f074d54a243bd148648f671 2023-07-22T12:26:31,755 INFO [StoreOpener-73a8d9b14b06f35bfa58d9fccd108576-1 {}] regionserver.HStore(324): Store=73a8d9b14b06f35bfa58d9fccd108576/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:26:31,755 INFO [StoreOpener-73a8d9b14b06f35bfa58d9fccd108576-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:31,756 INFO [StoreOpener-73a8d9b14b06f35bfa58d9fccd108576-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73a8d9b14b06f35bfa58d9fccd108576 columnFamilyName session 2023-07-22T12:26:31,756 DEBUG [StoreOpener-73a8d9b14b06f35bfa58d9fccd108576-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:26:31,762 DEBUG [StoreOpener-73a8d9b14b06f35bfa58d9fccd108576-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/764190440e7a41d18df8d98ff5230476 2023-07-22T12:26:31,766 DEBUG [StoreOpener-73a8d9b14b06f35bfa58d9fccd108576-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/ff0d1b4b15d542d786093c22d34f0ff7 2023-07-22T12:26:31,766 INFO [StoreOpener-73a8d9b14b06f35bfa58d9fccd108576-1 {}] regionserver.HStore(324): Store=73a8d9b14b06f35bfa58d9fccd108576/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:26:31,767 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:31,768 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:31,769 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-07-22T12:26:31,770 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1076): writing seq id for 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:31,771 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): Opened 73a8d9b14b06f35bfa58d9fccd108576; next sequenceid=35; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9622129280, jitterRate=-0.10386937856674194}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-07-22T12:26:31,772 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(992): Region open journal for 73a8d9b14b06f35bfa58d9fccd108576: 2023-07-22T12:26:31,773 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576., pid=53, masterSystemTime=1690028791742 2023-07-22T12:26:31,774 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:31,774 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(158): Opened backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:31,775 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=73a8d9b14b06f35bfa58d9fccd108576, regionState=OPEN, openSeqNum=35, regionLocation=jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:31,778 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=53, resume processing ppid=52 2023-07-22T12:26:31,778 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=53, ppid=52, state=SUCCESS; OpenRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576, server=jenkins-hbase3.apache.org,45275,1690028749134 in 186 msec 2023-07-22T12:26:31,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=52, resume processing ppid=51 2023-07-22T12:26:31,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=52, ppid=51, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=73a8d9b14b06f35bfa58d9fccd108576, ASSIGN in 343 msec 2023-07-22T12:26:31,780 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028791780"}]},"ts":"1690028791780"} 2023-07-22T12:26:31,782 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-07-22T12:26:31,783 INFO [PEWorker-4 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-07-22T12:26:31,785 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=51, state=SUCCESS; EnableTableProcedure table=backup:system in 357 msec 2023-07-22T12:26:32,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=51 2023-07-22T12:26:32,034 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1525): Done restoring backup system table 2023-07-22T12:26:32,034 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:26:32,048 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:26:32,061 WARN [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupAdminImpl(128): Backup system table snapshot exists 2023-07-22T12:26:32,064 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupAdminImpl(227): Deleting backup backup_1690028764280 ... 2023-07-22T12:26:32,066 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] util.BackupUtils(443): Trying to cleanup up target dir : backup_1690028764280 2023-07-22T12:26:32,066 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] util.BackupUtils(458): No data has been found in hdfs://localhost:35695/backupUT/backup_1690028764280/default/test-1690028758503. 2023-07-22T12:26:32,067 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] util.BackupUtils(465): hdfs://localhost:35695/backupUT/backup_1690028764280/default is empty, remove it. 2023-07-22T12:26:32,075 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupAdminImpl(269): 0 bulk loaded files out of 0 were deleted 2023-07-22T12:26:32,075 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(433): deleted 0 original bulkload rows 2023-07-22T12:26:32,076 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupAdminImpl(276): Delete backup backup_1690028764280 completed. 2023-07-22T12:26:32,079 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1014): test-1690028758503 2023-07-22T12:26:32,081 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:26:32,088 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:26:32,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:26:32,090 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:26:32,090 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(637): Finish backup exclusive operation DELETE operation finished OK: backup_1690028764280 No failed backup MERGE operation found 2023-07-22T12:26:32,093 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:26:32,096 ERROR [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1559): Snapshot snapshot_backup_system does not exists 2023-07-22T12:26:32,096 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:26:32,097 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:26:32,100 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] backup.TestBackupDeleteWithFailures(143): test repair backup delete on a single table with data and failures POST_DELETE_SNAPSHOT_FAILURE 2023-07-22T12:26:32,112 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78a5c9e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:26:32,116 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:26:32,118 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:50718, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:26:32,120 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:26:32,122 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:59016, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:26:32,134 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:26:32,139 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:26:32,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690028792139 (current time:1690028792139). 2023-07-22T12:26:32,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:26:32,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:26:32,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@450577e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:26:32,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:26:32,148 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:50728, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:26:32,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:26:32,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:26:32,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure2.ProcedureExecutor(1033): Stored pid=54, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:26:32,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 54 2023-07-22T12:26:32,152 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:26:32,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-07-22T12:26:32,153 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:26:32,154 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:26:32,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-07-22T12:26:32,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-07-22T12:26:32,561 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:26:32,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE; SnapshotRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576}] 2023-07-22T12:26:32,563 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=55, ppid=54, state=RUNNABLE; SnapshotRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:32,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=55 2023-07-22T12:26:32,715 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:32,716 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2819): Flushing 73a8d9b14b06f35bfa58d9fccd108576 2/2 column families, dataSize=417 B heapSize=1.49 KB 2023-07-22T12:26:32,733 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/meta/8cf43acf588b4068a36e7c3dc8960f53 is 91, key is incrbackupset:hdfs://localhost:35695/backupUT/meta:test-1690028758503/1690028792079/Put/seqid=0 2023-07-22T12:26:32,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-07-22T12:26:33,138 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=184 B at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/meta/8cf43acf588b4068a36e7c3dc8960f53 2023-07-22T12:26:33,144 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8cf43acf588b4068a36e7c3dc8960f53 2023-07-22T12:26:33,145 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8cf43acf588b4068a36e7c3dc8960f53 2023-07-22T12:26:33,160 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/session/bba70fd2f4344bc69ee098d84c05fe54 is 59, key is session:backup_1690028764280/session:/1690028792075/DeleteFamily/seqid=0 2023-07-22T12:26:33,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-07-22T12:26:33,516 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-07-22T12:26:33,516 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-07-22T12:26:33,565 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=233 B at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/session/bba70fd2f4344bc69ee098d84c05fe54 2023-07-22T12:26:33,571 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bba70fd2f4344bc69ee098d84c05fe54 2023-07-22T12:26:33,571 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: bba70fd2f4344bc69ee098d84c05fe54 2023-07-22T12:26:33,572 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/meta/8cf43acf588b4068a36e7c3dc8960f53 as hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/8cf43acf588b4068a36e7c3dc8960f53 2023-07-22T12:26:33,579 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8cf43acf588b4068a36e7c3dc8960f53 2023-07-22T12:26:33,579 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/8cf43acf588b4068a36e7c3dc8960f53, entries=2, sequenceid=45, filesize=5.2 K 2023-07-22T12:26:33,580 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/session/bba70fd2f4344bc69ee098d84c05fe54 as hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/bba70fd2f4344bc69ee098d84c05fe54 2023-07-22T12:26:33,585 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bba70fd2f4344bc69ee098d84c05fe54 2023-07-22T12:26:33,586 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/bba70fd2f4344bc69ee098d84c05fe54, entries=2, sequenceid=45, filesize=5.2 K 2023-07-22T12:26:33,586 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(3022): Finished flush of dataSize ~417 B/417, heapSize ~1.46 KB/1496, currentSize=0 B/0 for 73a8d9b14b06f35bfa58d9fccd108576 in 870ms, sequenceid=45, compaction requested=true 2023-07-22T12:26:33,587 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2520): Flush status journal for 73a8d9b14b06f35bfa58d9fccd108576: 2023-07-22T12:26:33,587 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. for snapshot_backup_system completed. 2023-07-22T12:26:33,587 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:26:33,587 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:26:33,587 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/214bafad8f074d54a243bd148648f671, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/8cf43acf588b4068a36e7c3dc8960f53] hfiles 2023-07-22T12:26:33,587 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/214bafad8f074d54a243bd148648f671 for snapshot=snapshot_backup_system 2023-07-22T12:26:33,587 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/8cf43acf588b4068a36e7c3dc8960f53 for snapshot=snapshot_backup_system 2023-07-22T12:26:33,587 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/764190440e7a41d18df8d98ff5230476, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/ff0d1b4b15d542d786093c22d34f0ff7, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/bba70fd2f4344bc69ee098d84c05fe54] hfiles 2023-07-22T12:26:33,587 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/764190440e7a41d18df8d98ff5230476 for snapshot=snapshot_backup_system 2023-07-22T12:26:33,587 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/ff0d1b4b15d542d786093c22d34f0ff7 for snapshot=snapshot_backup_system 2023-07-22T12:26:33,587 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/bba70fd2f4344bc69ee098d84c05fe54 for snapshot=snapshot_backup_system 2023-07-22T12:26:33,995 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:33,995 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=55 2023-07-22T12:26:33,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.HMaster(3994): Remote procedure done, pid=55 2023-07-22T12:26:33,996 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:33,997 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=55, ppid=54, state=RUNNABLE; SnapshotRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:33,999 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=55, resume processing ppid=54 2023-07-22T12:26:34,000 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:26:34,000 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=55, ppid=54, state=SUCCESS; SnapshotRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576 in 1.4370 sec 2023-07-22T12:26:34,000 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:26:34,001 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:26:34,001 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:26:34,002 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:26:34,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-07-22T12:26:34,415 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:26:34,422 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:26:34,423 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:26:34,839 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:26:34,839 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 54 2023-07-22T12:26:34,840 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=54, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.6890 sec 2023-07-22T12:26:36,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-07-22T12:26:36,258 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] impl.TableBackupClient(115): Backup backup_1690028792113 started at 1690028796258. 2023-07-22T12:26:36,260 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.TableBackupClient(119): Backup session backup_1690028792113 has been started. 2023-07-22T12:26:36,261 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-07-22T12:26:36,262 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-07-22T12:26:36,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-07-22T12:26:36,262 INFO [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-07-22T12:26:36,262 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-07-22T12:26:36,263 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-07-22T12:26:36,264 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-07-22T12:26:36,264 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:26:36,266 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:36,266 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:26:36,266 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:26:36,266 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-07-22T12:26:36,266 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-07-22T12:26:36,267 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-07-22T12:26:36,267 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:26:36,267 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-07-22T12:26:36,267 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-07-22T12:26:36,267 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-07-22T12:26:36,267 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-07-22T12:26:36,267 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-07-22T12:26:36,268 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-07-22T12:26:36,268 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-07-22T12:26:36,268 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,45275,1690028749134' joining acquired barrier for procedure (rolllog) in zk 2023-07-22T12:26:36,270 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:36,270 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-07-22T12:26:36,270 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:26:36,270 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:26:36,271 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:26:36,271 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-07-22T12:26:36,271 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:26:36,271 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:26:36,272 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:36,272 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:26:36,272 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,45275,1690028749134' joining acquired barrier for procedure 'rolllog' on coordinator 2023-07-22T12:26:36,272 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@3a1dd463[Count = 0] remaining members to acquire global barrier 2023-07-22T12:26:36,273 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-07-22T12:26:36,273 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-07-22T12:26:36,278 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-07-22T12:26:36,278 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-07-22T12:26:36,278 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-07-22T12:26:36,278 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-07-22T12:26:36,278 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-07-22T12:26:36,278 DEBUG [rs(jenkins-hbase3.apache.org,45275,1690028749134)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:36,278 INFO [rs(jenkins-hbase3.apache.org,45275,1690028749134)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1690028773016 highest: 1690028773016 on jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:36,278 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45275%2C1690028749134.meta:.meta(num 1690028772983) roll requested 2023-07-22T12:26:36,289 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.meta.1690028796278.meta, exclude list is [], retry=0 2023-07-22T12:26:36,295 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35109,DS-940ef494-c177-4702-ad53-b2959f1d3b03,DISK] 2023-07-22T12:26:36,297 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.meta.1690028772983.meta with entries=20, filesize=4.92 KB; new WAL /user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.meta.1690028796278.meta 2023-07-22T12:26:36,297 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:35109,DS-940ef494-c177-4702-ad53-b2959f1d3b03,DISK]] 2023-07-22T12:26:36,297 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.meta.1690028772983.meta is not closed yet, will try archiving it next time 2023-07-22T12:26:36,297 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45275%2C1690028749134:(num 1690028773016) roll requested 2023-07-22T12:26:36,300 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-07-22T12:26:36,300 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-07-22T12:26:36,309 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.1690028796297, exclude list is [], retry=0 2023-07-22T12:26:36,312 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35109,DS-940ef494-c177-4702-ad53-b2959f1d3b03,DISK] 2023-07-22T12:26:36,313 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.1690028773016 with entries=35, filesize=7.33 KB; new WAL /user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.1690028796297 2023-07-22T12:26:36,314 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:35109,DS-940ef494-c177-4702-ad53-b2959f1d3b03,DISK]] 2023-07-22T12:26:36,314 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.1690028773016 is not closed yet, will try archiving it next time 2023-07-22T12:26:36,315 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.1690028773016 not finished, retry = 0 2023-07-22T12:26:36,318 DEBUG [rs(jenkins-hbase3.apache.org,45275,1690028749134)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-07-22T12:26:36,318 INFO [rs(jenkins-hbase3.apache.org,45275,1690028749134)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1690028796297 on jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:36,328 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-07-22T12:26:36,328 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,45275,1690028749134' in zk 2023-07-22T12:26:36,330 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-07-22T12:26:36,330 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:26:36,330 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-07-22T12:26:36,330 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:36,331 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:26:36,331 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:26:36,331 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:26:36,331 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:26:36,332 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:26:36,332 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:36,332 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:26:36,333 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:26:36,333 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:36,333 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,45275,1690028749134': 2023-07-22T12:26:36,334 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,45275,1690028749134' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-07-22T12:26:36,334 INFO [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-07-22T12:26:36,334 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-07-22T12:26:36,334 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-07-22T12:26:36,334 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-07-22T12:26:36,334 INFO [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-07-22T12:26:36,335 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-07-22T12:26:36,335 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-07-22T12:26:36,335 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:26:36,335 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:26:36,335 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:26:36,336 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:26:36,336 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:26:36,336 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:26:36,336 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:26:36,336 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:26:36,337 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:26:36,337 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:26:36,337 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:36,338 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:26:36,338 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:26:36,339 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:36,341 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:26:36,341 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:26:36,341 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-07-22T12:26:36,341 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:26:36,341 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-07-22T12:26:36,342 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:26:36,342 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:26:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-07-22T12:26:36,446 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1690028796445_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } 2023-07-22T12:26:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690028796446 (current time:1690028796446). 2023-07-22T12:26:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:26:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1690028796445_default_test-1690028758503 VERSION not specified, setting to 2 2023-07-22T12:26:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@247c44ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:26:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:26:36,456 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:50734, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:26:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:26:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:26:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure2.ProcedureExecutor(1033): Stored pid=56, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1690028796445_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } 2023-07-22T12:26:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1690028796445_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 }, snapshot procedure id = 56 2023-07-22T12:26:36,460 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1690028796445_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:26:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=56 2023-07-22T12:26:36,461 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1690028796445_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:26:36,462 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1690028796445_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:26:36,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=56 2023-07-22T12:26:36,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=56 2023-07-22T12:26:36,868 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1690028796445_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:26:36,868 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE; SnapshotRegionProcedure 648e0182e924cbe38699c76e79cee0a5}] 2023-07-22T12:26:36,870 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=57, ppid=56, state=RUNNABLE; SnapshotRegionProcedure 648e0182e924cbe38699c76e79cee0a5 2023-07-22T12:26:37,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=57 2023-07-22T12:26:37,022 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5. 2023-07-22T12:26:37,022 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HRegion(2520): Flush status journal for 648e0182e924cbe38699c76e79cee0a5: 2023-07-22T12:26:37,022 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5. for snapshot_1690028796445_default_test-1690028758503 completed. 2023-07-22T12:26:37,022 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(240): Storing 'test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5.' region-info for snapshot=snapshot_1690028796445_default_test-1690028758503 2023-07-22T12:26:37,022 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:26:37,022 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/default/test-1690028758503/648e0182e924cbe38699c76e79cee0a5/f/3c8f99d0de094dbb9a6f3cb1dadc3824] hfiles 2023-07-22T12:26:37,022 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/default/test-1690028758503/648e0182e924cbe38699c76e79cee0a5/f/3c8f99d0de094dbb9a6f3cb1dadc3824 for snapshot=snapshot_1690028796445_default_test-1690028758503 2023-07-22T12:26:37,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=56 2023-07-22T12:26:37,249 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:26:37,428 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5. 2023-07-22T12:26:37,428 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=57 2023-07-22T12:26:37,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.HMaster(3994): Remote procedure done, pid=57 2023-07-22T12:26:37,429 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1690028796445_default_test-1690028758503 on region 648e0182e924cbe38699c76e79cee0a5 2023-07-22T12:26:37,430 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=57, ppid=56, state=RUNNABLE; SnapshotRegionProcedure 648e0182e924cbe38699c76e79cee0a5 2023-07-22T12:26:37,432 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=57, resume processing ppid=56 2023-07-22T12:26:37,433 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=57, ppid=56, state=SUCCESS; SnapshotRegionProcedure 648e0182e924cbe38699c76e79cee0a5 in 563 msec 2023-07-22T12:26:37,433 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1690028796445_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:26:37,433 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1690028796445_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:26:37,434 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1690028796445_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:26:37,434 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1690028796445_default_test-1690028758503 2023-07-22T12:26:37,435 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/.tmp/snapshot_1690028796445_default_test-1690028758503 2023-07-22T12:26:37,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=56 2023-07-22T12:26:37,844 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1690028796445_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:26:37,850 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1690028796445_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:26:37,850 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/.tmp/snapshot_1690028796445_default_test-1690028758503 to hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/snapshot_1690028796445_default_test-1690028758503 2023-07-22T12:26:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=56 2023-07-22T12:26:38,668 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1690028796445_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:26:38,668 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1690028796445_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 }, snapshot procedure id = 56 2023-07-22T12:26:38,669 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=56, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1690028796445_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } in 2.2100 sec 2023-07-22T12:26:40,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=56 2023-07-22T12:26:40,567 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1690028792113 2023-07-22T12:26:40,568 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-07-22T12:26:40,568 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-07-22T12:26:40,568 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1690028792113_test-1690028758503 2023-07-22T12:26:40,568 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1690028796445_default_test-1690028758503 to hdfs://localhost:35695/backupUT/backup_1690028792113/default/test-1690028758503/ 2023-07-22T12:26:40,568 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-07-22T12:26:40,601 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:35695, inputRoot=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb 2023-07-22T12:26:40,601 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,34453,1690028746140_-515723105_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:35695/backupUT/backup_1690028792113/default/test-1690028758503, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:35695/backupUT/backup_1690028792113/default/test-1690028758503/.hbase-snapshot/.tmp/snapshot_1690028796445_default_test-1690028758503 2023-07-22T12:26:40,601 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:35695, inputRoot=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb, snapshotDir=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/snapshot_1690028796445_default_test-1690028758503. 2023-07-22T12:26:40,609 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/snapshot_1690028796445_default_test-1690028758503 to hdfs://localhost:35695/backupUT/backup_1690028792113/default/test-1690028758503/.hbase-snapshot/.tmp/snapshot_1690028796445_default_test-1690028758503 2023-07-22T12:26:41,025 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:41,025 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:41,026 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:41,026 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:42,046 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/hadoop-6583246014605098295.jar 2023-07-22T12:26:42,047 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:42,047 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:42,047 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:42,048 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:42,048 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:42,049 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:42,049 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:42,049 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:42,050 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:42,051 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:26:42,051 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:26:42,052 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:26:42,052 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:26:42,053 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:26:42,053 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:26:42,054 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:26:42,054 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:26:42,054 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:26:42,055 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:26:42,055 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:26:42,056 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:26:42,057 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:26:42,057 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:26:42,058 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:26:42,058 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:26:42,059 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:26:42,059 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:26:42,060 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:26:42,105 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1690028796445_default_test-1690028758503' hfile list 2023-07-22T12:26:42,108 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-07-22T12:26:42,215 WARN [Listener at jenkins-hbase3.apache.org/43519 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/mapreduce.cluster.local.dir/1690028802159/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:26:42,216 WARN [Listener at jenkins-hbase3.apache.org/43519 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/mapreduce.cluster.local.dir/1690028802159/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:26:42,270 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-07-22T12:26:42,295 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/default/test-1690028758503/648e0182e924cbe38699c76e79cee0a5/f/3c8f99d0de094dbb9a6f3cb1dadc3824 output=hdfs://localhost:35695/backupUT/backup_1690028792113/default/test-1690028758503/archive/data/default/test-1690028758503/648e0182e924cbe38699c76e79cee0a5/f/3c8f99d0de094dbb9a6f3cb1dadc3824 2023-07-22T12:26:42,295 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-07-22T12:26:43,252 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-07-22T12:26:43,254 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-07-22T12:26:43,260 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1690028796445_default_test-1690028758503 2023-07-22T12:26:43,260 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1690028796445_default_test-1690028758503 finished. 2023-07-22T12:26:43,260 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1014): test-1690028758503 2023-07-22T12:26:43,267 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690028792113 2023-07-22T12:26:43,267 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-07-22T12:26:43,672 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:35695/backupUT/backup_1690028792113/.backup.manifest 2023-07-22T12:26:43,673 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.TableBackupClient(358): Backup backup_1690028792113 finished: type=FULL,tablelist=test-1690028758503,targetRootDir=hdfs://localhost:35695/backupUT,startts=1690028796258,completets=1690028803267,bytescopied=0 2023-07-22T12:26:43,673 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-07-22T12:26:43,673 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1690028796445_default_test-1690028758503 2023-07-22T12:26:43,674 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1690028796445_default_test-1690028758503" 2023-07-22T12:26:43,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1690028796445_default_test-1690028758503 2023-07-22T12:26:43,677 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1690028796445_default_test-1690028758503 for backup backup_1690028792113 succeeded. 2023-07-22T12:26:43,678 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:26:43,684 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:26:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:26:43,687 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:26:43,689 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:26:43,694 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] impl.TableBackupClient(376): Backup backup_1690028792113 completed. 2023-07-22T12:26:43,695 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:26:43,707 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] backup.TestBackupDeleteWithFailures(147): backup complete 2023-07-22T12:26:43,728 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:26:43,740 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:26:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690028803740 (current time:1690028803740). 2023-07-22T12:26:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:26:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:26:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa45397, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:26:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:26:43,748 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52644, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:26:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:26:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:26:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure2.ProcedureExecutor(1033): Stored pid=58, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:26:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 58 2023-07-22T12:26:43,751 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:26:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=58 2023-07-22T12:26:43,752 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:26:43,754 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:26:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=58 2023-07-22T12:26:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=58 2023-07-22T12:26:44,164 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:26:44,165 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE; SnapshotRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576}] 2023-07-22T12:26:44,167 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=59, ppid=58, state=RUNNABLE; SnapshotRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:44,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=59 2023-07-22T12:26:44,318 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:44,319 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegion(2819): Flushing 73a8d9b14b06f35bfa58d9fccd108576 2/2 column families, dataSize=1.06 KB heapSize=2.22 KB 2023-07-22T12:26:44,323 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/meta/a1d8a297b4004f2f86d039b91f30c845 is 169, key is trslm:hdfs://localhost:35695/backupUT\x00test-1690028758503/meta:log-roll-map/1690028803263/Put/seqid=0 2023-07-22T12:26:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=58 2023-07-22T12:26:44,728 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=511 B at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/meta/a1d8a297b4004f2f86d039b91f30c845 2023-07-22T12:26:44,733 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a1d8a297b4004f2f86d039b91f30c845 2023-07-22T12:26:44,736 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/session/100d749ceddc4d57aac29277e58ee440 is 310, key is session:backup_1690028792113/session:context/1690028803688/Put/seqid=0 2023-07-22T12:26:44,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=58 2023-07-22T12:26:45,141 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=579 B at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/session/100d749ceddc4d57aac29277e58ee440 2023-07-22T12:26:45,147 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 100d749ceddc4d57aac29277e58ee440 2023-07-22T12:26:45,148 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/meta/a1d8a297b4004f2f86d039b91f30c845 as hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/a1d8a297b4004f2f86d039b91f30c845 2023-07-22T12:26:45,153 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/a1d8a297b4004f2f86d039b91f30c845, entries=5, sequenceid=57, filesize=5.5 K 2023-07-22T12:26:45,154 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/session/100d749ceddc4d57aac29277e58ee440 as hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/100d749ceddc4d57aac29277e58ee440 2023-07-22T12:26:45,159 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/100d749ceddc4d57aac29277e58ee440, entries=2, sequenceid=57, filesize=5.3 K 2023-07-22T12:26:45,160 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegion(3022): Finished flush of dataSize ~1.06 KB/1090, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 73a8d9b14b06f35bfa58d9fccd108576 in 842ms, sequenceid=57, compaction requested=true 2023-07-22T12:26:45,160 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegion(2520): Flush status journal for 73a8d9b14b06f35bfa58d9fccd108576: 2023-07-22T12:26:45,160 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. for snapshot_backup_system completed. 2023-07-22T12:26:45,160 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:26:45,160 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:26:45,160 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/214bafad8f074d54a243bd148648f671, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/8cf43acf588b4068a36e7c3dc8960f53, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/a1d8a297b4004f2f86d039b91f30c845] hfiles 2023-07-22T12:26:45,160 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/214bafad8f074d54a243bd148648f671 for snapshot=snapshot_backup_system 2023-07-22T12:26:45,160 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/8cf43acf588b4068a36e7c3dc8960f53 for snapshot=snapshot_backup_system 2023-07-22T12:26:45,160 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/a1d8a297b4004f2f86d039b91f30c845 for snapshot=snapshot_backup_system 2023-07-22T12:26:45,160 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/764190440e7a41d18df8d98ff5230476, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/ff0d1b4b15d542d786093c22d34f0ff7, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/bba70fd2f4344bc69ee098d84c05fe54, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/100d749ceddc4d57aac29277e58ee440] hfiles 2023-07-22T12:26:45,160 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/764190440e7a41d18df8d98ff5230476 for snapshot=snapshot_backup_system 2023-07-22T12:26:45,161 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/ff0d1b4b15d542d786093c22d34f0ff7 for snapshot=snapshot_backup_system 2023-07-22T12:26:45,161 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/bba70fd2f4344bc69ee098d84c05fe54 for snapshot=snapshot_backup_system 2023-07-22T12:26:45,161 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/100d749ceddc4d57aac29277e58ee440 for snapshot=snapshot_backup_system 2023-07-22T12:26:45,566 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:45,567 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=59 2023-07-22T12:26:45,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.HMaster(3994): Remote procedure done, pid=59 2023-07-22T12:26:45,567 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:45,568 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=59, ppid=58, state=RUNNABLE; SnapshotRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:45,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=59, resume processing ppid=58 2023-07-22T12:26:45,571 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:26:45,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=59, ppid=58, state=SUCCESS; SnapshotRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576 in 1.4040 sec 2023-07-22T12:26:45,571 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:26:45,572 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:26:45,572 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:26:45,574 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:26:45,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=58 2023-07-22T12:26:45,984 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:26:45,994 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:26:45,995 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:26:46,816 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:26:46,816 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 58 2023-07-22T12:26:46,817 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=58, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0670 sec 2023-07-22T12:26:47,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=58 2023-07-22T12:26:47,865 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupAdminImpl(227): Deleting backup backup_1690028792113 ... 2023-07-22T12:26:47,865 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] util.BackupUtils(435): Delete log files: .backup.manifest 2023-07-22T12:26:47,866 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] util.BackupUtils(435): Delete log files: default 2023-07-22T12:26:47,867 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] util.BackupUtils(443): Trying to cleanup up target dir : backup_1690028792113 2023-07-22T12:26:47,867 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] util.BackupUtils(458): No data has been found in hdfs://localhost:35695/backupUT/backup_1690028792113/default/test-1690028758503. 2023-07-22T12:26:47,868 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] util.BackupUtils(465): hdfs://localhost:35695/backupUT/backup_1690028792113/default is empty, remove it. 2023-07-22T12:26:47,871 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupAdminImpl(269): 0 bulk loaded files out of 0 were deleted 2023-07-22T12:26:47,871 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(433): deleted 0 original bulkload rows 2023-07-22T12:26:47,873 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupAdminImpl(276): Delete backup backup_1690028792113 completed. 2023-07-22T12:26:47,880 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1014): test-1690028758503 2023-07-22T12:26:47,882 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:26:47,889 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:26:47,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:26:47,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.MetricsHBaseServer(144): Unknown exception type java.io.IOException: postDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.postDeleteSnapshot(TestBackupDeleteWithFailures.java:111) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$89.call(MasterCoprocessorHost.java:1164) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$89.call(MasterCoprocessorHost.java:1161) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.postDeleteSnapshot(MasterCoprocessorHost.java:1161) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:386) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:760) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-07-22T12:26:47,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.CallRunner(138): callId: 282 service: MasterService methodName: DeleteSnapshot size: 57 connection: 172.31.12.81:46424 deadline: 1690028867889, exception=java.io.IOException: postDeleteSnapshot 2023-07-22T12:26:47,993 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:26:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.MetricsHBaseServer(144): Unknown exception type org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException: Snapshot 'snapshot_backup_system' doesn't exist on the filesystem at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:359) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:760) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-07-22T12:26:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.CallRunner(138): callId: 284 service: MasterService methodName: DeleteSnapshot size: 57 connection: 172.31.12.81:46424 deadline: 1690028867993, exception=org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException: Snapshot 'snapshot_backup_system' doesn't exist on the filesystem 2023-07-22T12:26:47,999 WARN [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupAdminImpl(163): Delete operation succeeded, there were some errors: org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException: org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException: Snapshot 'snapshot_backup_system' doesn't exist on the filesystem at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:359) at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:760) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.MasterCallable.call(MasterCallable.java:101) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3018) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3010) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.deleteSnapshot(HBaseAdmin.java:2918) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.backup.impl.BackupSystemTable.deleteSnapshot(BackupSystemTable.java:1556) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.deleteBackups(BackupAdminImpl.java:149) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures.testBackupDeleteWithFailuresAfter(TestBackupDeleteWithFailures.java:161) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures.testBackupDeleteWithFailures(TestBackupDeleteWithFailures.java:137) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException: Snapshot 'snapshot_backup_system' doesn't exist on the filesystem at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:359) at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:760) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:388) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:92) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:420) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:114) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:129) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:165) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:195) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] ... 1 more 2023-07-22T12:26:48,001 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:26:48,014 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e7caaf7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:26:48,022 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f11a6fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:26:48,023 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:26:48,025 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:59298, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:26:48,028 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:26:48,029 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52650, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService REPAIR status: no failed sessions found. Checking failed delete backup operation ... No failed backup DELETE operation found 2023-07-22T12:26:48,046 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:26:48,050 ERROR [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1559): Snapshot snapshot_backup_system does not exists No failed backup MERGE operation found 2023-07-22T12:26:48,051 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:26:48,055 ERROR [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1559): Snapshot snapshot_backup_system does not exists 2023-07-22T12:26:48,055 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:26:48,055 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:26:48,059 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] backup.TestBackupDeleteWithFailures(143): test repair backup delete on a single table with data and failures PRE_SNAPSHOT_FAILURE 2023-07-22T12:26:48,068 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fc07230, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:26:48,161 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:26:48,162 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52654, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:26:48,164 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:26:48,165 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:59304, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:26:48,176 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:26:48,182 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:26:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690028808182 (current time:1690028808182). 2023-07-22T12:26:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:26:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:26:48,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@627f980, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:26:48,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:26:48,189 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52660, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:26:48,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:26:48,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:26:48,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure2.ProcedureExecutor(1033): Stored pid=60, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:26:48,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2023-07-22T12:26:48,193 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:26:48,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-07-22T12:26:48,194 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:26:48,195 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:26:48,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-07-22T12:26:48,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-07-22T12:26:48,606 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:26:48,606 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE; SnapshotRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576}] 2023-07-22T12:26:48,608 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=61, ppid=60, state=RUNNABLE; SnapshotRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:48,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=61 2023-07-22T12:26:48,760 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:48,760 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2819): Flushing 73a8d9b14b06f35bfa58d9fccd108576 2/2 column families, dataSize=268 B heapSize=1.13 KB 2023-07-22T12:26:48,764 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/meta/baa225541baa43e49bfda9352d92c627 is 91, key is incrbackupset:hdfs://localhost:35695/backupUT/meta:test-1690028758503/1690028807881/Put/seqid=0 2023-07-22T12:26:48,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-07-22T12:26:49,169 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=124 B at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/meta/baa225541baa43e49bfda9352d92c627 2023-07-22T12:26:49,175 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for baa225541baa43e49bfda9352d92c627 2023-07-22T12:26:49,175 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: baa225541baa43e49bfda9352d92c627 2023-07-22T12:26:49,178 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/session/d82946dfb50a4195a831393cc6556d32 is 59, key is session:backup_1690028792113/session:/1690028807872/DeleteFamily/seqid=0 2023-07-22T12:26:49,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-07-22T12:26:49,583 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=144 B at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/session/d82946dfb50a4195a831393cc6556d32 2023-07-22T12:26:49,588 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d82946dfb50a4195a831393cc6556d32 2023-07-22T12:26:49,588 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: d82946dfb50a4195a831393cc6556d32 2023-07-22T12:26:49,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/meta/baa225541baa43e49bfda9352d92c627 as hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/baa225541baa43e49bfda9352d92c627 2023-07-22T12:26:49,594 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for baa225541baa43e49bfda9352d92c627 2023-07-22T12:26:49,594 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/baa225541baa43e49bfda9352d92c627, entries=2, sequenceid=65, filesize=5.2 K 2023-07-22T12:26:49,595 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/session/d82946dfb50a4195a831393cc6556d32 as hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/d82946dfb50a4195a831393cc6556d32 2023-07-22T12:26:49,601 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d82946dfb50a4195a831393cc6556d32 2023-07-22T12:26:49,601 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/d82946dfb50a4195a831393cc6556d32, entries=2, sequenceid=65, filesize=5.2 K 2023-07-22T12:26:49,602 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(3022): Finished flush of dataSize ~268 B/268, heapSize ~1.09 KB/1120, currentSize=0 B/0 for 73a8d9b14b06f35bfa58d9fccd108576 in 842ms, sequenceid=65, compaction requested=true 2023-07-22T12:26:49,602 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2520): Flush status journal for 73a8d9b14b06f35bfa58d9fccd108576: 2023-07-22T12:26:49,602 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. for snapshot_backup_system completed. 2023-07-22T12:26:49,602 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:26:49,602 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:26:49,602 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/214bafad8f074d54a243bd148648f671, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/8cf43acf588b4068a36e7c3dc8960f53, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/a1d8a297b4004f2f86d039b91f30c845, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/baa225541baa43e49bfda9352d92c627] hfiles 2023-07-22T12:26:49,602 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/214bafad8f074d54a243bd148648f671 for snapshot=snapshot_backup_system 2023-07-22T12:26:49,602 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/8cf43acf588b4068a36e7c3dc8960f53 for snapshot=snapshot_backup_system 2023-07-22T12:26:49,602 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/a1d8a297b4004f2f86d039b91f30c845 for snapshot=snapshot_backup_system 2023-07-22T12:26:49,602 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/baa225541baa43e49bfda9352d92c627 for snapshot=snapshot_backup_system 2023-07-22T12:26:49,602 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/764190440e7a41d18df8d98ff5230476, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/ff0d1b4b15d542d786093c22d34f0ff7, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/bba70fd2f4344bc69ee098d84c05fe54, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/100d749ceddc4d57aac29277e58ee440, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/d82946dfb50a4195a831393cc6556d32] hfiles 2023-07-22T12:26:49,602 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/764190440e7a41d18df8d98ff5230476 for snapshot=snapshot_backup_system 2023-07-22T12:26:49,602 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/ff0d1b4b15d542d786093c22d34f0ff7 for snapshot=snapshot_backup_system 2023-07-22T12:26:49,602 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/bba70fd2f4344bc69ee098d84c05fe54 for snapshot=snapshot_backup_system 2023-07-22T12:26:49,602 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/100d749ceddc4d57aac29277e58ee440 for snapshot=snapshot_backup_system 2023-07-22T12:26:49,602 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/d82946dfb50a4195a831393cc6556d32 for snapshot=snapshot_backup_system 2023-07-22T12:26:49,609 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:49,609 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=61 2023-07-22T12:26:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.HMaster(3994): Remote procedure done, pid=61 2023-07-22T12:26:49,610 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:49,611 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=61, ppid=60, state=RUNNABLE; SnapshotRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:49,613 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=61, resume processing ppid=60 2023-07-22T12:26:49,613 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:26:49,613 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=61, ppid=60, state=SUCCESS; SnapshotRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576 in 1.0050 sec 2023-07-22T12:26:49,613 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:26:49,614 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:26:49,614 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:26:49,615 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:26:50,027 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:26:50,037 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:26:50,038 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:26:50,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-07-22T12:26:50,457 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:26:50,457 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2023-07-22T12:26:50,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=60, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2660 sec 2023-07-22T12:26:51,452 DEBUG [regionserver/jenkins-hbase3:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.15384615384615385, tune throughput to 57.69 MB/second 2023-07-22T12:26:52,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-07-22T12:26:52,303 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] impl.TableBackupClient(115): Backup backup_1690028808068 started at 1690028812302. 2023-07-22T12:26:52,304 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.TableBackupClient(119): Backup session backup_1690028808068 has been started. 2023-07-22T12:26:52,306 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-07-22T12:26:52,307 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-07-22T12:26:52,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-07-22T12:26:52,307 INFO [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-07-22T12:26:52,307 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-07-22T12:26:52,308 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-07-22T12:26:52,308 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-07-22T12:26:52,309 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:26:52,310 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:52,310 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:26:52,310 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:26:52,310 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-07-22T12:26:52,310 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-07-22T12:26:52,311 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-07-22T12:26:52,311 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:26:52,311 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-07-22T12:26:52,311 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-07-22T12:26:52,311 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-07-22T12:26:52,311 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-07-22T12:26:52,311 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-07-22T12:26:52,312 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-07-22T12:26:52,313 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-07-22T12:26:52,313 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,45275,1690028749134' joining acquired barrier for procedure (rolllog) in zk 2023-07-22T12:26:52,315 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-07-22T12:26:52,315 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:52,315 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:26:52,315 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:26:52,315 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-07-22T12:26:52,315 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:26:52,316 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:26:52,316 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:26:52,316 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:52,316 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:26:52,317 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,45275,1690028749134' joining acquired barrier for procedure 'rolllog' on coordinator 2023-07-22T12:26:52,317 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@79eda685[Count = 0] remaining members to acquire global barrier 2023-07-22T12:26:52,317 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-07-22T12:26:52,317 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-07-22T12:26:52,319 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-07-22T12:26:52,319 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-07-22T12:26:52,319 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-07-22T12:26:52,319 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-07-22T12:26:52,319 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-07-22T12:26:52,319 DEBUG [rs(jenkins-hbase3.apache.org,45275,1690028749134)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:52,319 INFO [rs(jenkins-hbase3.apache.org,45275,1690028749134)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1690028796297 highest: 1690028796297 on jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:52,319 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45275%2C1690028749134.meta:.meta(num 1690028796278) roll requested 2023-07-22T12:26:52,330 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.meta.1690028812319.meta, exclude list is [], retry=0 2023-07-22T12:26:52,335 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35109,DS-940ef494-c177-4702-ad53-b2959f1d3b03,DISK] 2023-07-22T12:26:52,337 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.meta.1690028796278.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.meta.1690028812319.meta 2023-07-22T12:26:52,337 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:35109,DS-940ef494-c177-4702-ad53-b2959f1d3b03,DISK]] 2023-07-22T12:26:52,337 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.meta.1690028796278.meta is not closed yet, will try archiving it next time 2023-07-22T12:26:52,337 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C45275%2C1690028749134:(num 1690028796297) roll requested 2023-07-22T12:26:52,340 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.meta.1690028796278.meta to hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/oldWALs/jenkins-hbase3.apache.org%2C45275%2C1690028749134.meta.1690028796278.meta 2023-07-22T12:26:52,349 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.1690028812337, exclude list is [], retry=0 2023-07-22T12:26:52,352 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35109,DS-940ef494-c177-4702-ad53-b2959f1d3b03,DISK] 2023-07-22T12:26:52,353 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.1690028796297 with entries=18, filesize=3.44 KB; new WAL /user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.1690028812337 2023-07-22T12:26:52,353 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:35109,DS-940ef494-c177-4702-ad53-b2959f1d3b03,DISK]] 2023-07-22T12:26:52,353 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.1690028796297 is not closed yet, will try archiving it next time 2023-07-22T12:26:52,354 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.1690028773016 to hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/oldWALs/jenkins-hbase3.apache.org%2C45275%2C1690028749134.1690028773016 2023-07-22T12:26:52,359 DEBUG [rs(jenkins-hbase3.apache.org,45275,1690028749134)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-07-22T12:26:52,359 INFO [rs(jenkins-hbase3.apache.org,45275,1690028749134)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1690028812337 on jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:52,368 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-07-22T12:26:52,368 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,45275,1690028749134' in zk 2023-07-22T12:26:52,370 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-07-22T12:26:52,370 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:26:52,370 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:52,370 DEBUG [member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-07-22T12:26:52,371 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:26:52,371 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:26:52,371 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:26:52,372 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:26:52,372 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:26:52,372 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:52,372 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:26:52,373 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:26:52,373 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:52,374 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,45275,1690028749134': 2023-07-22T12:26:52,374 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,45275,1690028749134' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-07-22T12:26:52,374 INFO [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-07-22T12:26:52,374 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-07-22T12:26:52,374 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-07-22T12:26:52,374 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-07-22T12:26:52,374 INFO [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-07-22T12:26:52,375 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-07-22T12:26:52,375 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-07-22T12:26:52,376 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:26:52,376 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:26:52,376 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:26:52,376 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:26:52,376 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:26:52,376 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:26:52,376 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:26:52,376 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:26:52,377 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:26:52,377 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:26:52,377 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:52,378 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:26:52,378 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:26:52,378 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:26:52,381 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:26:52,382 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:26:52,382 DEBUG [(jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:26:52,382 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-07-22T12:26:52,382 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-07-22T12:26:52,382 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:26:52,382 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:26:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-07-22T12:26:52,486 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1690028812485_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } 2023-07-22T12:26:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690028812486 (current time:1690028812486). 2023-07-22T12:26:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:26:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1690028812485_default_test-1690028758503 VERSION not specified, setting to 2 2023-07-22T12:26:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@517dc167, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:26:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:26:52,497 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:54166, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:26:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:26:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:26:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure2.ProcedureExecutor(1033): Stored pid=62, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1690028812485_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } 2023-07-22T12:26:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1690028812485_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-07-22T12:26:52,501 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1690028812485_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:26:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-07-22T12:26:52,502 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1690028812485_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:26:52,503 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1690028812485_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:26:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-07-22T12:26:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-07-22T12:26:52,909 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1690028812485_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:26:52,909 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure 648e0182e924cbe38699c76e79cee0a5}] 2023-07-22T12:26:52,913 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure 648e0182e924cbe38699c76e79cee0a5 2023-07-22T12:26:53,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=63 2023-07-22T12:26:53,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5. 2023-07-22T12:26:53,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegion(2520): Flush status journal for 648e0182e924cbe38699c76e79cee0a5: 2023-07-22T12:26:53,064 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5. for snapshot_1690028812485_default_test-1690028758503 completed. 2023-07-22T12:26:53,065 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(240): Storing 'test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5.' region-info for snapshot=snapshot_1690028812485_default_test-1690028758503 2023-07-22T12:26:53,065 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:26:53,065 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/default/test-1690028758503/648e0182e924cbe38699c76e79cee0a5/f/3c8f99d0de094dbb9a6f3cb1dadc3824] hfiles 2023-07-22T12:26:53,065 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/default/test-1690028758503/648e0182e924cbe38699c76e79cee0a5/f/3c8f99d0de094dbb9a6f3cb1dadc3824 for snapshot=snapshot_1690028812485_default_test-1690028758503 2023-07-22T12:26:53,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-07-22T12:26:53,471 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5. 2023-07-22T12:26:53,471 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=63 2023-07-22T12:26:53,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.HMaster(3994): Remote procedure done, pid=63 2023-07-22T12:26:53,471 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1690028812485_default_test-1690028758503 on region 648e0182e924cbe38699c76e79cee0a5 2023-07-22T12:26:53,473 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure 648e0182e924cbe38699c76e79cee0a5 2023-07-22T12:26:53,476 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=63, resume processing ppid=62 2023-07-22T12:26:53,476 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=63, ppid=62, state=SUCCESS; SnapshotRegionProcedure 648e0182e924cbe38699c76e79cee0a5 in 565 msec 2023-07-22T12:26:53,476 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1690028812485_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:26:53,476 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1690028812485_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:26:53,477 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1690028812485_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:26:53,477 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1690028812485_default_test-1690028758503 2023-07-22T12:26:53,478 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/.tmp/snapshot_1690028812485_default_test-1690028758503 2023-07-22T12:26:53,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-07-22T12:26:53,888 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1690028812485_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:26:53,893 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1690028812485_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:26:53,893 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/.tmp/snapshot_1690028812485_default_test-1690028758503 to hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/snapshot_1690028812485_default_test-1690028758503 2023-07-22T12:26:54,042 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 281efe51d339c53c39818444fc3ed1c9 changed from -1.0 to 0.0, refreshing cache 2023-07-22T12:26:54,312 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1690028812485_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:26:54,312 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1690028812485_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-07-22T12:26:54,314 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=62, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1690028812485_default_test-1690028758503 table=test-1690028758503 type=FLUSH ttl=0 } in 1.8140 sec 2023-07-22T12:26:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-07-22T12:26:54,606 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1690028808068 2023-07-22T12:26:54,606 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-07-22T12:26:54,606 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-07-22T12:26:54,606 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1690028808068_test-1690028758503 2023-07-22T12:26:54,606 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1690028812485_default_test-1690028758503 to hdfs://localhost:35695/backupUT/backup_1690028808068/default/test-1690028758503/ 2023-07-22T12:26:54,606 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-07-22T12:26:54,634 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:35695, inputRoot=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb 2023-07-22T12:26:54,634 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,34453,1690028746140_-1030560570_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:35695/backupUT/backup_1690028808068/default/test-1690028758503, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:35695/backupUT/backup_1690028808068/default/test-1690028758503/.hbase-snapshot/.tmp/snapshot_1690028812485_default_test-1690028758503 2023-07-22T12:26:54,634 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:35695, inputRoot=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb, snapshotDir=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/snapshot_1690028812485_default_test-1690028758503. 2023-07-22T12:26:54,644 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/snapshot_1690028812485_default_test-1690028758503 to hdfs://localhost:35695/backupUT/backup_1690028808068/default/test-1690028758503/.hbase-snapshot/.tmp/snapshot_1690028812485_default_test-1690028758503 2023-07-22T12:26:55,058 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:55,059 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:55,060 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:55,061 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:56,073 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/hadoop-4711138900064528308.jar 2023-07-22T12:26:56,074 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:56,074 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:56,075 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:56,076 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:56,076 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:56,077 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:56,077 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:56,078 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:56,079 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:26:56,080 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:26:56,081 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:26:56,082 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:26:56,082 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:26:56,083 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:26:56,084 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:26:56,084 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:26:56,085 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:26:56,086 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:26:56,086 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:26:56,087 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:26:56,087 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:26:56,088 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:26:56,089 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:26:56,089 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:26:56,090 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:26:56,090 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:26:56,091 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:26:56,092 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:26:56,138 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1690028812485_default_test-1690028758503' hfile list 2023-07-22T12:26:56,140 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-07-22T12:26:56,226 WARN [Listener at jenkins-hbase3.apache.org/43519 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/mapreduce.cluster.local.dir/1690028816185/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:26:56,226 WARN [Listener at jenkins-hbase3.apache.org/43519 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/mapreduce.cluster.local.dir/1690028816185/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:26:56,284 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-07-22T12:26:56,330 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/default/test-1690028758503/648e0182e924cbe38699c76e79cee0a5/f/3c8f99d0de094dbb9a6f3cb1dadc3824 output=hdfs://localhost:35695/backupUT/backup_1690028808068/default/test-1690028758503/archive/data/default/test-1690028758503/648e0182e924cbe38699c76e79cee0a5/f/3c8f99d0de094dbb9a6f3cb1dadc3824 2023-07-22T12:26:56,330 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-07-22T12:26:57,266 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-07-22T12:26:57,267 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-07-22T12:26:57,274 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1690028812485_default_test-1690028758503 2023-07-22T12:26:57,274 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1690028812485_default_test-1690028758503 finished. 2023-07-22T12:26:57,274 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1014): test-1690028758503 2023-07-22T12:26:57,280 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690028808068 2023-07-22T12:26:57,281 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-07-22T12:26:57,686 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:35695/backupUT/backup_1690028808068/.backup.manifest 2023-07-22T12:26:57,686 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.TableBackupClient(358): Backup backup_1690028808068 finished: type=FULL,tablelist=test-1690028758503,targetRootDir=hdfs://localhost:35695/backupUT,startts=1690028812302,completets=1690028817280,bytescopied=0 2023-07-22T12:26:57,686 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-07-22T12:26:57,686 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1690028812485_default_test-1690028758503 2023-07-22T12:26:57,687 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1690028812485_default_test-1690028758503" 2023-07-22T12:26:57,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1690028812485_default_test-1690028758503 2023-07-22T12:26:57,691 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1690028812485_default_test-1690028758503 for backup backup_1690028808068 succeeded. 2023-07-22T12:26:57,692 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:26:57,699 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:26:57,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:26:57,702 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:26:57,706 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:26:57,713 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] impl.TableBackupClient(376): Backup backup_1690028808068 completed. 2023-07-22T12:26:57,713 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:26:57,729 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] backup.TestBackupDeleteWithFailures(147): backup complete 2023-07-22T12:26:57,767 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:26:57,781 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:26:57,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690028817781 (current time:1690028817781). 2023-07-22T12:26:57,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:26:57,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:26:57,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@673d40ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:26:57,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:26:57,792 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:54176, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:26:57,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:26:57,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:26:57,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure2.ProcedureExecutor(1033): Stored pid=64, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:26:57,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-07-22T12:26:57,796 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:26:57,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-07-22T12:26:57,797 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:26:57,799 WARN [PEWorker-1 {}] procedure.SnapshotProcedure(200): unexpected exception while execute pid=64, state=FAILED:SNAPSHOT_PRE_OPERATION, locked=true, exception=java.io.IOException via master-snapshot:java.io.IOException: preSnapshot; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }. Mark procedure Failed. java.io.IOException: preSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preSnapshot(TestBackupDeleteWithFailures.java:95) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$79.call(MasterCoprocessorHost.java:1068) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$79.call(MasterCoprocessorHost.java:1065) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preSnapshot(MasterCoprocessorHost.java:1065) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:306) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1650) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1396) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1962) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1989) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] 2023-07-22T12:26:57,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1532): Rolled back pid=64, state=ROLLEDBACK, exception=java.io.IOException via master-snapshot:java.io.IOException: preSnapshot; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } exec-time=24 msec 2023-07-22T12:26:57,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-07-22T12:26:57,913 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10d3701c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:26:57,918 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@788b4d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:26:57,918 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:26:57,920 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:43912, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:26:57,924 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:26:57,926 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:54184, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService REPAIR status: no failed sessions found. Checking failed delete backup operation ... Found failed DELETE operation for: backup_1690028808068 Running DELETE again ... 2023-07-22T12:26:57,941 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-07-22T12:26:57,946 WARN [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1530): Could not restore backup system table. Snapshot snapshot_backup_system does not exists. 2023-07-22T12:26:57,946 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:26:57,958 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:26:57,969 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:26:57,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690028817969 (current time:1690028817969). 2023-07-22T12:26:57,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:26:57,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:26:57,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c611c98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:26:57,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:26:57,982 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:54186, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:26:57,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:26:57,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:26:57,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure2.ProcedureExecutor(1033): Stored pid=65, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:26:57,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 65 2023-07-22T12:26:57,986 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:26:57,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=65 2023-07-22T12:26:57,986 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:26:57,988 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:26:58,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=65 2023-07-22T12:26:58,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=65 2023-07-22T12:26:58,401 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:26:58,402 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE; SnapshotRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576}] 2023-07-22T12:26:58,403 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=66, ppid=65, state=RUNNABLE; SnapshotRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:58,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45275 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=66 2023-07-22T12:26:58,555 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:58,555 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.HRegion(2819): Flushing 73a8d9b14b06f35bfa58d9fccd108576 2/2 column families, dataSize=1.21 KB heapSize=2.59 KB 2023-07-22T12:26:58,561 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/meta/d0719b322db3485695a49a7dc46c4cb9 is 169, key is trslm:hdfs://localhost:35695/backupUT\x00test-1690028758503/meta:log-roll-map/1690028817277/Put/seqid=0 2023-07-22T12:26:58,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=65 2023-07-22T12:26:58,965 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=571 B at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/meta/d0719b322db3485695a49a7dc46c4cb9 2023-07-22T12:26:58,971 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: d0719b322db3485695a49a7dc46c4cb9 2023-07-22T12:26:58,974 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/session/4a103ef65a41469a81345bb071fc6f86 is 310, key is session:backup_1690028808068/session:context/1690028817702/Put/seqid=0 2023-07-22T12:26:59,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=65 2023-07-22T12:26:59,378 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=668 B at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/session/4a103ef65a41469a81345bb071fc6f86 2023-07-22T12:26:59,383 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 4a103ef65a41469a81345bb071fc6f86 2023-07-22T12:26:59,384 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/meta/d0719b322db3485695a49a7dc46c4cb9 as hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/d0719b322db3485695a49a7dc46c4cb9 2023-07-22T12:26:59,388 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/d0719b322db3485695a49a7dc46c4cb9, entries=5, sequenceid=80, filesize=5.5 K 2023-07-22T12:26:59,389 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/session/4a103ef65a41469a81345bb071fc6f86 as hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/4a103ef65a41469a81345bb071fc6f86 2023-07-22T12:26:59,393 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/4a103ef65a41469a81345bb071fc6f86, entries=2, sequenceid=80, filesize=5.3 K 2023-07-22T12:26:59,394 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.HRegion(3022): Finished flush of dataSize ~1.21 KB/1239, heapSize ~2.55 KB/2616, currentSize=0 B/0 for 73a8d9b14b06f35bfa58d9fccd108576 in 839ms, sequenceid=80, compaction requested=true 2023-07-22T12:26:59,394 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.HRegion(2520): Flush status journal for 73a8d9b14b06f35bfa58d9fccd108576: 2023-07-22T12:26:59,394 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. for snapshot_backup_system completed. 2023-07-22T12:26:59,394 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:26:59,394 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:26:59,394 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/214bafad8f074d54a243bd148648f671, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/8cf43acf588b4068a36e7c3dc8960f53, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/a1d8a297b4004f2f86d039b91f30c845, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/baa225541baa43e49bfda9352d92c627, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/d0719b322db3485695a49a7dc46c4cb9] hfiles 2023-07-22T12:26:59,395 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/214bafad8f074d54a243bd148648f671 for snapshot=snapshot_backup_system 2023-07-22T12:26:59,395 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/8cf43acf588b4068a36e7c3dc8960f53 for snapshot=snapshot_backup_system 2023-07-22T12:26:59,395 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/a1d8a297b4004f2f86d039b91f30c845 for snapshot=snapshot_backup_system 2023-07-22T12:26:59,395 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/baa225541baa43e49bfda9352d92c627 for snapshot=snapshot_backup_system 2023-07-22T12:26:59,395 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/d0719b322db3485695a49a7dc46c4cb9 for snapshot=snapshot_backup_system 2023-07-22T12:26:59,395 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/764190440e7a41d18df8d98ff5230476, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/ff0d1b4b15d542d786093c22d34f0ff7, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/bba70fd2f4344bc69ee098d84c05fe54, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/100d749ceddc4d57aac29277e58ee440, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/d82946dfb50a4195a831393cc6556d32, hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/4a103ef65a41469a81345bb071fc6f86] hfiles 2023-07-22T12:26:59,395 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (1/6): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/764190440e7a41d18df8d98ff5230476 for snapshot=snapshot_backup_system 2023-07-22T12:26:59,395 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (2/6): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/ff0d1b4b15d542d786093c22d34f0ff7 for snapshot=snapshot_backup_system 2023-07-22T12:26:59,395 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (3/6): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/bba70fd2f4344bc69ee098d84c05fe54 for snapshot=snapshot_backup_system 2023-07-22T12:26:59,395 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (4/6): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/100d749ceddc4d57aac29277e58ee440 for snapshot=snapshot_backup_system 2023-07-22T12:26:59,395 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (5/6): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/d82946dfb50a4195a831393cc6556d32 for snapshot=snapshot_backup_system 2023-07-22T12:26:59,395 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (6/6): hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/4a103ef65a41469a81345bb071fc6f86 for snapshot=snapshot_backup_system 2023-07-22T12:26:59,800 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:26:59,800 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=66 2023-07-22T12:26:59,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.HMaster(3994): Remote procedure done, pid=66 2023-07-22T12:26:59,801 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:59,802 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=66, ppid=65, state=RUNNABLE; SnapshotRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:26:59,804 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=66, resume processing ppid=65 2023-07-22T12:26:59,804 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:26:59,805 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=66, ppid=65, state=SUCCESS; SnapshotRegionProcedure 73a8d9b14b06f35bfa58d9fccd108576 in 1.4010 sec 2023-07-22T12:26:59,805 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:26:59,806 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:26:59,806 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:26:59,809 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:27:00,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=65 2023-07-22T12:27:00,219 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:27:00,230 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:27:00,230 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:27:01,048 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:27:01,048 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 65 2023-07-22T12:27:01,049 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=65, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0650 sec 2023-07-22T12:27:02,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=65 2023-07-22T12:27:02,099 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupAdminImpl(227): Deleting backup backup_1690028808068 ... 2023-07-22T12:27:02,100 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] util.BackupUtils(435): Delete log files: .backup.manifest 2023-07-22T12:27:02,100 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] util.BackupUtils(435): Delete log files: default 2023-07-22T12:27:02,101 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] util.BackupUtils(443): Trying to cleanup up target dir : backup_1690028808068 2023-07-22T12:27:02,101 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] util.BackupUtils(458): No data has been found in hdfs://localhost:35695/backupUT/backup_1690028808068/default/test-1690028758503. 2023-07-22T12:27:02,102 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] util.BackupUtils(465): hdfs://localhost:35695/backupUT/backup_1690028808068/default is empty, remove it. 2023-07-22T12:27:02,108 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupAdminImpl(269): 0 bulk loaded files out of 0 were deleted 2023-07-22T12:27:02,108 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(433): deleted 0 original bulkload rows 2023-07-22T12:27:02,110 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupAdminImpl(276): Delete backup backup_1690028808068 completed. 2023-07-22T12:27:02,116 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1014): test-1690028758503 2023-07-22T12:27:02,118 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:27:02,125 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:27:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:27:02,127 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:27:02,127 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(637): Finish backup exclusive operation DELETE operation finished OK: backup_1690028808068 No failed backup MERGE operation found 2023-07-22T12:27:02,130 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:27:02,133 ERROR [Listener at jenkins-hbase3.apache.org/43519 {}] impl.BackupSystemTable(1559): Snapshot snapshot_backup_system does not exists 2023-07-22T12:27:02,133 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:27:02,134 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:27:02,155 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] hbase.ResourceChecker(175): after: backup.TestBackupDeleteWithFailures#testBackupDeleteWithFailures Thread=593 (was 583) Potentially hanging thread: rs(jenkins-hbase3.apache.org,45275,1690028749134)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_444246832_16 at /127.0.0.1:42604 [Receiving block BP-1668548271-172.31.12.81-1690028742924:blk_1073741924_1100] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:60489@0x54b3e235-SendThread(127.0.0.1:60489) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1261) Potentially hanging thread: hconnection-0x7d45fd63-shared-pool-31 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1003790371_16 at /127.0.0.1:42648 [Waiting for operation #14] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/cluster_5c7a65c4-cab7-9c85-3939-092136c4beb7/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:60489@0x54b3e235 java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1455) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1485) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$494/3737444.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: region-location-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1668548271-172.31.12.81-1690028742924:blk_1073741923_1099, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1003790371_16 at /127.0.0.1:42616 [Waiting for operation #23] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/cluster_5c7a65c4-cab7-9c85-3939-092136c4beb7/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:60489@0x24197a5a-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:60489@0x24197a5a-SendThread(127.0.0.1:60489) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:228) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1338) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1261) Potentially hanging thread: member: 'jenkins-hbase3.apache.org,45275,1690028749134' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_444246832_16 at /127.0.0.1:42594 [Receiving block BP-1668548271-172.31.12.81-1690028742924:blk_1073741923_1099] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: (jenkins-hbase3.apache.org,34453,1690028746140)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:60489@0x24197a5a java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1455) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1485) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$494/3737444.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,45275,1690028749134)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1668548271-172.31.12.81-1690028742924:blk_1073741924_1100, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:60489@0x54b3e235-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: rs(jenkins-hbase3.apache.org,45275,1690028749134)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1758989841_16 at /127.0.0.1:42744 [Waiting for operation #14] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=680 (was 660) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=123 (was 202), ProcessCount=168 (was 169), AvailableMemoryMB=16675 (was 17309) 2023-07-22T12:27:02,155 WARN [Listener at jenkins-hbase3.apache.org/43519 {}] hbase.ResourceChecker(130): Thread=593 is superior to 500 2023-07-22T12:27:02,162 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1690028783878" 2023-07-22T12:27:02,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34453 {}] snapshot.SnapshotManager(378): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1690028783878 2023-07-22T12:27:02,166 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-07-22T12:27:02,166 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:27:02,166 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-07-22T12:27:02,166 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] util.JVMClusterUtil(257): Found active master hash=1662072280, stopped=false 2023-07-22T12:27:02,167 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver 2023-07-22T12:27:02,167 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] master.ServerManager(910): Cluster shutdown requested of master=jenkins-hbase3.apache.org,34453,1690028746140 2023-07-22T12:27:02,169 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] procedure2.ProcedureExecutor(631): Stopping 2023-07-22T12:27:02,170 DEBUG [Listener at jenkins-hbase3.apache.org/43519 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:27:02,170 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] regionserver.HRegionServer(2513): ***** STOPPING region server 'jenkins-hbase3.apache.org,45275,1690028749134' ***** 2023-07-22T12:27:02,170 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] regionserver.HRegionServer(2527): STOPPED: Shutdown requested 2023-07-22T12:27:02,170 INFO [RS:0;jenkins-hbase3:45275 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-07-22T12:27:02,180 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-07-22T12:27:02,184 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-07-22T12:27:02,185 INFO [RS:0;jenkins-hbase3:45275 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@40aab786{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-07-22T12:27:02,189 INFO [RS:0;jenkins-hbase3:45275 {}] server.AbstractConnector(383): Stopped ServerConnector@135ebf2f{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-07-22T12:27:02,189 INFO [RS:0;jenkins-hbase3:45275 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-07-22T12:27:02,190 INFO [RS:0;jenkins-hbase3:45275 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@5fe3a699{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-07-22T12:27:02,190 INFO [RS:0;jenkins-hbase3:45275 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@5c2bf6c1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/hadoop.log.dir/,STOPPED} 2023-07-22T12:27:02,193 INFO [RS:0;jenkins-hbase3:45275 {}] regionserver.HeapMemoryManager(220): Stopping 2023-07-22T12:27:02,193 INFO [RS:0;jenkins-hbase3:45275 {}] flush.RegionServerFlushTableProcedureManager(117): Stopping region server flush procedure manager gracefully. 2023-07-22T12:27:02,193 INFO [RS:0;jenkins-hbase3:45275 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-07-22T12:27:02,193 INFO [RS:0;jenkins-hbase3:45275 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-07-22T12:27:02,193 INFO [RS:0;jenkins-hbase3:45275 {}] regionserver.HRegionServer(3524): Received CLOSE for 01adde20a336e8a75d65a8603dd7f6c0 2023-07-22T12:27:02,194 INFO [RS:0;jenkins-hbase3:45275 {}] regionserver.HRegionServer(3524): Received CLOSE for ed8e6284cfcb2ae38b4da50cf240aee9 2023-07-22T12:27:02,194 INFO [RS:0;jenkins-hbase3:45275 {}] regionserver.HRegionServer(3524): Received CLOSE for 73a8d9b14b06f35bfa58d9fccd108576 2023-07-22T12:27:02,194 INFO [RS:0;jenkins-hbase3:45275 {}] regionserver.HRegionServer(3524): Received CLOSE for 40f3d448196e452f47f760cf6a94ef80 2023-07-22T12:27:02,194 INFO [RS:0;jenkins-hbase3:45275 {}] regionserver.HRegionServer(3524): Received CLOSE for 281efe51d339c53c39818444fc3ed1c9 2023-07-22T12:27:02,194 INFO [RS:0;jenkins-hbase3:45275 {}] regionserver.HRegionServer(3524): Received CLOSE for 648e0182e924cbe38699c76e79cee0a5 2023-07-22T12:27:02,194 INFO [RS:0;jenkins-hbase3:45275 {}] regionserver.HRegionServer(3524): Received CLOSE for 999a9c03f7b1cabe862a9a76f7912808 2023-07-22T12:27:02,194 INFO [RS:0;jenkins-hbase3:45275 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,45275,1690028749134 2023-07-22T12:27:02,194 DEBUG [RS:0;jenkins-hbase3:45275 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:27:02,195 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 01adde20a336e8a75d65a8603dd7f6c0, disabling compactions & flushes 2023-07-22T12:27:02,195 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-16900287585032,,1690028760985.01adde20a336e8a75d65a8603dd7f6c0. 2023-07-22T12:27:02,195 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16900287585032,,1690028760985.01adde20a336e8a75d65a8603dd7f6c0. 2023-07-22T12:27:02,195 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16900287585032,,1690028760985.01adde20a336e8a75d65a8603dd7f6c0. after waiting 0 ms 2023-07-22T12:27:02,195 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16900287585032,,1690028760985.01adde20a336e8a75d65a8603dd7f6c0. 2023-07-22T12:27:02,194 INFO [RS:0;jenkins-hbase3:45275 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-07-22T12:27:02,195 INFO [RS:0;jenkins-hbase3:45275 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-07-22T12:27:02,195 INFO [RS:0;jenkins-hbase3:45275 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-07-22T12:27:02,195 INFO [RS:0;jenkins-hbase3:45275 {}] regionserver.HRegionServer(3524): Received CLOSE for 1588230740 2023-07-22T12:27:02,196 INFO [RS:0;jenkins-hbase3:45275 {}] regionserver.HRegionServer(1579): Waiting on 8 regions to close 2023-07-22T12:27:02,196 DEBUG [RS:0;jenkins-hbase3:45275 {}] regionserver.HRegionServer(1583): Online Regions={01adde20a336e8a75d65a8603dd7f6c0=ns3:test-16900287585032,,1690028760985.01adde20a336e8a75d65a8603dd7f6c0., ed8e6284cfcb2ae38b4da50cf240aee9=ns4:test-16900287585033,,1690028762109.ed8e6284cfcb2ae38b4da50cf240aee9., 73a8d9b14b06f35bfa58d9fccd108576=backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576., 1588230740=hbase:meta,,1.1588230740, 40f3d448196e452f47f760cf6a94ef80=ns2:test-16900287585031,,1690028760257.40f3d448196e452f47f760cf6a94ef80., 281efe51d339c53c39818444fc3ed1c9=hbase:namespace,,1690028752470.281efe51d339c53c39818444fc3ed1c9., 648e0182e924cbe38699c76e79cee0a5=test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5., 999a9c03f7b1cabe862a9a76f7912808=backup:system_bulk,,1690028766564.999a9c03f7b1cabe862a9a76f7912808.} 2023-07-22T12:27:02,196 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-07-22T12:27:02,196 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-07-22T12:27:02,196 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-07-22T12:27:02,196 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-07-22T12:27:02,196 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-07-22T12:27:02,197 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=13.22 KB heapSize=23.11 KB 2023-07-22T12:27:02,197 DEBUG [RS:0;jenkins-hbase3:45275 {}] regionserver.HRegionServer(1609): Waiting on 01adde20a336e8a75d65a8603dd7f6c0, 1588230740, 281efe51d339c53c39818444fc3ed1c9, 40f3d448196e452f47f760cf6a94ef80, 648e0182e924cbe38699c76e79cee0a5, 73a8d9b14b06f35bfa58d9fccd108576, 999a9c03f7b1cabe862a9a76f7912808, ed8e6284cfcb2ae38b4da50cf240aee9 2023-07-22T12:27:02,200 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/ns3/test-16900287585032/01adde20a336e8a75d65a8603dd7f6c0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-07-22T12:27:02,201 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:27:02,202 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-16900287585032,,1690028760985.01adde20a336e8a75d65a8603dd7f6c0. 2023-07-22T12:27:02,202 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 01adde20a336e8a75d65a8603dd7f6c0: 2023-07-22T12:27:02,202 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-16900287585032,,1690028760985.01adde20a336e8a75d65a8603dd7f6c0. 2023-07-22T12:27:02,203 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing ed8e6284cfcb2ae38b4da50cf240aee9, disabling compactions & flushes 2023-07-22T12:27:02,203 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-16900287585033,,1690028762109.ed8e6284cfcb2ae38b4da50cf240aee9. 2023-07-22T12:27:02,203 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16900287585033,,1690028762109.ed8e6284cfcb2ae38b4da50cf240aee9. 2023-07-22T12:27:02,203 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16900287585033,,1690028762109.ed8e6284cfcb2ae38b4da50cf240aee9. after waiting 0 ms 2023-07-22T12:27:02,203 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16900287585033,,1690028762109.ed8e6284cfcb2ae38b4da50cf240aee9. 2023-07-22T12:27:02,212 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/ns4/test-16900287585033/ed8e6284cfcb2ae38b4da50cf240aee9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-07-22T12:27:02,213 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:27:02,214 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-16900287585033,,1690028762109.ed8e6284cfcb2ae38b4da50cf240aee9. 2023-07-22T12:27:02,214 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for ed8e6284cfcb2ae38b4da50cf240aee9: 2023-07-22T12:27:02,214 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-16900287585033,,1690028762109.ed8e6284cfcb2ae38b4da50cf240aee9. 2023-07-22T12:27:02,214 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 73a8d9b14b06f35bfa58d9fccd108576, disabling compactions & flushes 2023-07-22T12:27:02,214 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:27:02,214 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:27:02,214 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. after waiting 0 ms 2023-07-22T12:27:02,214 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:27:02,214 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 73a8d9b14b06f35bfa58d9fccd108576 2/2 column families, dataSize=223 B heapSize=1.01 KB 2023-07-22T12:27:02,219 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/meta/aaf645f110154df487730d3bd0b27a96 is 91, key is incrbackupset:hdfs://localhost:35695/backupUT/meta:test-1690028758503/1690028822116/Put/seqid=0 2023-07-22T12:27:02,224 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/hbase/meta/1588230740/.tmp/info/6ba36a1a10954864a3e22cefcf74a9b3 is 159, key is ns2:test-16900287585031,,1690028760257.40f3d448196e452f47f760cf6a94ef80./info:regioninfo/1690028760703/Put/seqid=0 2023-07-22T12:27:02,231 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=124 B at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/meta/aaf645f110154df487730d3bd0b27a96 2023-07-22T12:27:02,238 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for aaf645f110154df487730d3bd0b27a96 2023-07-22T12:27:02,238 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: aaf645f110154df487730d3bd0b27a96 2023-07-22T12:27:02,241 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/session/25b2e87d34fe496d8c7cd007cfcedef3 is 59, key is session:backup_1690028808068/session:/1690028822109/DeleteFamily/seqid=0 2023-07-22T12:27:02,397 DEBUG [RS:0;jenkins-hbase3:45275 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 281efe51d339c53c39818444fc3ed1c9, 40f3d448196e452f47f760cf6a94ef80, 648e0182e924cbe38699c76e79cee0a5, 73a8d9b14b06f35bfa58d9fccd108576, 999a9c03f7b1cabe862a9a76f7912808 2023-07-22T12:27:02,457 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-07-22T12:27:02,457 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-07-22T12:27:02,597 DEBUG [RS:0;jenkins-hbase3:45275 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 281efe51d339c53c39818444fc3ed1c9, 40f3d448196e452f47f760cf6a94ef80, 648e0182e924cbe38699c76e79cee0a5, 73a8d9b14b06f35bfa58d9fccd108576, 999a9c03f7b1cabe862a9a76f7912808 2023-07-22T12:27:02,633 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.18 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/hbase/meta/1588230740/.tmp/info/6ba36a1a10954864a3e22cefcf74a9b3 2023-07-22T12:27:02,639 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6ba36a1a10954864a3e22cefcf74a9b3 2023-07-22T12:27:02,648 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=99 B at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/session/25b2e87d34fe496d8c7cd007cfcedef3 2023-07-22T12:27:02,654 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 25b2e87d34fe496d8c7cd007cfcedef3 2023-07-22T12:27:02,655 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 25b2e87d34fe496d8c7cd007cfcedef3 2023-07-22T12:27:02,655 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/hbase/meta/1588230740/.tmp/table/1cd70f05d73044c4a5a93dd0e1ed5d92 is 59, key is ns2:test-16900287585031/table:state/1690028760713/Put/seqid=0 2023-07-22T12:27:02,656 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/meta/aaf645f110154df487730d3bd0b27a96 as hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/aaf645f110154df487730d3bd0b27a96 2023-07-22T12:27:02,662 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for aaf645f110154df487730d3bd0b27a96 2023-07-22T12:27:02,662 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/meta/aaf645f110154df487730d3bd0b27a96, entries=2, sequenceid=87, filesize=5.2 K 2023-07-22T12:27:02,663 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/.tmp/session/25b2e87d34fe496d8c7cd007cfcedef3 as hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/25b2e87d34fe496d8c7cd007cfcedef3 2023-07-22T12:27:02,668 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 25b2e87d34fe496d8c7cd007cfcedef3 2023-07-22T12:27:02,668 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/session/25b2e87d34fe496d8c7cd007cfcedef3, entries=2, sequenceid=87, filesize=5.2 K 2023-07-22T12:27:02,669 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~223 B/223, heapSize ~1000 B/1000, currentSize=0 B/0 for 73a8d9b14b06f35bfa58d9fccd108576 in 455ms, sequenceid=87, compaction requested=true 2023-07-22T12:27:02,671 DEBUG [StoreFileCloser-73a8d9b14b06f35bfa58d9fccd108576-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 214bafad8f074d54a243bd148648f671 2023-07-22T12:27:02,672 DEBUG [StoreFileCloser-73a8d9b14b06f35bfa58d9fccd108576-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8cf43acf588b4068a36e7c3dc8960f53 2023-07-22T12:27:02,674 DEBUG [StoreFileCloser-73a8d9b14b06f35bfa58d9fccd108576-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a1d8a297b4004f2f86d039b91f30c845 2023-07-22T12:27:02,675 DEBUG [StoreFileCloser-73a8d9b14b06f35bfa58d9fccd108576-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: baa225541baa43e49bfda9352d92c627 2023-07-22T12:27:02,676 DEBUG [StoreFileCloser-73a8d9b14b06f35bfa58d9fccd108576-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: d0719b322db3485695a49a7dc46c4cb9 2023-07-22T12:27:02,678 DEBUG [StoreFileCloser-73a8d9b14b06f35bfa58d9fccd108576-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: aaf645f110154df487730d3bd0b27a96 2023-07-22T12:27:02,680 DEBUG [StoreFileCloser-73a8d9b14b06f35bfa58d9fccd108576-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 764190440e7a41d18df8d98ff5230476 2023-07-22T12:27:02,681 DEBUG [StoreFileCloser-73a8d9b14b06f35bfa58d9fccd108576-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: ff0d1b4b15d542d786093c22d34f0ff7 2023-07-22T12:27:02,683 DEBUG [StoreFileCloser-73a8d9b14b06f35bfa58d9fccd108576-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: bba70fd2f4344bc69ee098d84c05fe54 2023-07-22T12:27:02,685 DEBUG [StoreFileCloser-73a8d9b14b06f35bfa58d9fccd108576-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 100d749ceddc4d57aac29277e58ee440 2023-07-22T12:27:02,688 DEBUG [StoreFileCloser-73a8d9b14b06f35bfa58d9fccd108576-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: d82946dfb50a4195a831393cc6556d32 2023-07-22T12:27:02,693 DEBUG [StoreFileCloser-73a8d9b14b06f35bfa58d9fccd108576-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 4a103ef65a41469a81345bb071fc6f86 2023-07-22T12:27:02,693 DEBUG [StoreFileCloser-73a8d9b14b06f35bfa58d9fccd108576-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 25b2e87d34fe496d8c7cd007cfcedef3 2023-07-22T12:27:02,697 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system/73a8d9b14b06f35bfa58d9fccd108576/recovered.edits/90.seqid, newMaxSeqId=90, maxSeqId=34 2023-07-22T12:27:02,698 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:27:02,699 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:27:02,699 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 73a8d9b14b06f35bfa58d9fccd108576: 2023-07-22T12:27:02,699 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1690028764432.73a8d9b14b06f35bfa58d9fccd108576. 2023-07-22T12:27:02,700 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 40f3d448196e452f47f760cf6a94ef80, disabling compactions & flushes 2023-07-22T12:27:02,700 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-16900287585031,,1690028760257.40f3d448196e452f47f760cf6a94ef80. 2023-07-22T12:27:02,700 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16900287585031,,1690028760257.40f3d448196e452f47f760cf6a94ef80. 2023-07-22T12:27:02,700 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16900287585031,,1690028760257.40f3d448196e452f47f760cf6a94ef80. after waiting 0 ms 2023-07-22T12:27:02,700 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16900287585031,,1690028760257.40f3d448196e452f47f760cf6a94ef80. 2023-07-22T12:27:02,700 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 40f3d448196e452f47f760cf6a94ef80 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-07-22T12:27:02,715 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/ns2/test-16900287585031/40f3d448196e452f47f760cf6a94ef80/.tmp/f/d1502b32eba74bd8b96df096872c78e4 is 37, key is row10/f:q1/1690028760883/Put/seqid=0 2023-07-22T12:27:02,798 DEBUG [RS:0;jenkins-hbase3:45275 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 281efe51d339c53c39818444fc3ed1c9, 40f3d448196e452f47f760cf6a94ef80, 648e0182e924cbe38699c76e79cee0a5, 999a9c03f7b1cabe862a9a76f7912808 2023-07-22T12:27:02,998 DEBUG [RS:0;jenkins-hbase3:45275 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 281efe51d339c53c39818444fc3ed1c9, 40f3d448196e452f47f760cf6a94ef80, 648e0182e924cbe38699c76e79cee0a5, 999a9c03f7b1cabe862a9a76f7912808 2023-07-22T12:27:03,061 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/hbase/meta/1588230740/.tmp/table/1cd70f05d73044c4a5a93dd0e1ed5d92 2023-07-22T12:27:03,068 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 1cd70f05d73044c4a5a93dd0e1ed5d92 2023-07-22T12:27:03,069 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/hbase/meta/1588230740/.tmp/info/6ba36a1a10954864a3e22cefcf74a9b3 as hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/hbase/meta/1588230740/info/6ba36a1a10954864a3e22cefcf74a9b3 2023-07-22T12:27:03,082 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/hbase/meta/1588230740/info/6ba36a1a10954864a3e22cefcf74a9b3, entries=72, sequenceid=59, filesize=14.3 K 2023-07-22T12:27:03,083 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/hbase/meta/1588230740/.tmp/table/1cd70f05d73044c4a5a93dd0e1ed5d92 as hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/hbase/meta/1588230740/table/1cd70f05d73044c4a5a93dd0e1ed5d92 2023-07-22T12:27:03,091 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/hbase/meta/1588230740/table/1cd70f05d73044c4a5a93dd0e1ed5d92, entries=15, sequenceid=59, filesize=5.8 K 2023-07-22T12:27:03,092 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~13.22 KB/13542, heapSize ~22.83 KB/23376, currentSize=0 B/0 for 1588230740 in 894ms, sequenceid=59, compaction requested=false 2023-07-22T12:27:03,093 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6ba36a1a10954864a3e22cefcf74a9b3 2023-07-22T12:27:03,095 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 1cd70f05d73044c4a5a93dd0e1ed5d92 2023-07-22T12:27:03,099 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/hbase/meta/1588230740/recovered.edits/62.seqid, newMaxSeqId=62, maxSeqId=1 2023-07-22T12:27:03,100 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:27:03,100 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-07-22T12:27:03,101 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-07-22T12:27:03,101 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-07-22T12:27:03,101 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-07-22T12:27:03,119 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/ns2/test-16900287585031/40f3d448196e452f47f760cf6a94ef80/.tmp/f/d1502b32eba74bd8b96df096872c78e4 2023-07-22T12:27:03,125 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: d1502b32eba74bd8b96df096872c78e4 2023-07-22T12:27:03,126 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/ns2/test-16900287585031/40f3d448196e452f47f760cf6a94ef80/.tmp/f/d1502b32eba74bd8b96df096872c78e4 as hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/ns2/test-16900287585031/40f3d448196e452f47f760cf6a94ef80/f/d1502b32eba74bd8b96df096872c78e4 2023-07-22T12:27:03,131 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/ns2/test-16900287585031/40f3d448196e452f47f760cf6a94ef80/f/d1502b32eba74bd8b96df096872c78e4, entries=99, sequenceid=103, filesize=8.2 K 2023-07-22T12:27:03,132 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 40f3d448196e452f47f760cf6a94ef80 in 432ms, sequenceid=103, compaction requested=false 2023-07-22T12:27:03,134 DEBUG [StoreFileCloser-40f3d448196e452f47f760cf6a94ef80-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: d1502b32eba74bd8b96df096872c78e4 2023-07-22T12:27:03,139 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/ns2/test-16900287585031/40f3d448196e452f47f760cf6a94ef80/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-07-22T12:27:03,140 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:27:03,140 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-16900287585031,,1690028760257.40f3d448196e452f47f760cf6a94ef80. 2023-07-22T12:27:03,140 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 40f3d448196e452f47f760cf6a94ef80: 2023-07-22T12:27:03,140 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-16900287585031,,1690028760257.40f3d448196e452f47f760cf6a94ef80. 2023-07-22T12:27:03,141 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 281efe51d339c53c39818444fc3ed1c9, disabling compactions & flushes 2023-07-22T12:27:03,141 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1690028752470.281efe51d339c53c39818444fc3ed1c9. 2023-07-22T12:27:03,141 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1690028752470.281efe51d339c53c39818444fc3ed1c9. 2023-07-22T12:27:03,141 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1690028752470.281efe51d339c53c39818444fc3ed1c9. after waiting 0 ms 2023-07-22T12:27:03,141 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1690028752470.281efe51d339c53c39818444fc3ed1c9. 2023-07-22T12:27:03,141 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 281efe51d339c53c39818444fc3ed1c9 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-07-22T12:27:03,154 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/hbase/namespace/281efe51d339c53c39818444fc3ed1c9/.tmp/info/be428cd16bdf45c29c26db66cb435ecf is 45, key is default/info:d/1690028753449/Put/seqid=0 2023-07-22T12:27:03,162 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/hbase/namespace/281efe51d339c53c39818444fc3ed1c9/.tmp/info/be428cd16bdf45c29c26db66cb435ecf 2023-07-22T12:27:03,168 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: be428cd16bdf45c29c26db66cb435ecf 2023-07-22T12:27:03,168 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/hbase/namespace/281efe51d339c53c39818444fc3ed1c9/.tmp/info/be428cd16bdf45c29c26db66cb435ecf as hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/hbase/namespace/281efe51d339c53c39818444fc3ed1c9/info/be428cd16bdf45c29c26db66cb435ecf 2023-07-22T12:27:03,174 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/hbase/namespace/281efe51d339c53c39818444fc3ed1c9/info/be428cd16bdf45c29c26db66cb435ecf, entries=7, sequenceid=11, filesize=5.1 K 2023-07-22T12:27:03,175 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 281efe51d339c53c39818444fc3ed1c9 in 34ms, sequenceid=11, compaction requested=false 2023-07-22T12:27:03,178 DEBUG [StoreFileCloser-281efe51d339c53c39818444fc3ed1c9-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: be428cd16bdf45c29c26db66cb435ecf 2023-07-22T12:27:03,183 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/hbase/namespace/281efe51d339c53c39818444fc3ed1c9/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-07-22T12:27:03,184 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:27:03,185 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1690028752470.281efe51d339c53c39818444fc3ed1c9. 2023-07-22T12:27:03,185 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 281efe51d339c53c39818444fc3ed1c9: 2023-07-22T12:27:03,185 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1690028752470.281efe51d339c53c39818444fc3ed1c9. 2023-07-22T12:27:03,185 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 648e0182e924cbe38699c76e79cee0a5, disabling compactions & flushes 2023-07-22T12:27:03,185 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5. 2023-07-22T12:27:03,185 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5. 2023-07-22T12:27:03,185 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5. after waiting 0 ms 2023-07-22T12:27:03,185 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5. 2023-07-22T12:27:03,186 DEBUG [StoreFileCloser-648e0182e924cbe38699c76e79cee0a5-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3c8f99d0de094dbb9a6f3cb1dadc3824 2023-07-22T12:27:03,189 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/default/test-1690028758503/648e0182e924cbe38699c76e79cee0a5/recovered.edits/108.seqid, newMaxSeqId=108, maxSeqId=1 2023-07-22T12:27:03,190 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:27:03,191 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5. 2023-07-22T12:27:03,191 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 648e0182e924cbe38699c76e79cee0a5: 2023-07-22T12:27:03,191 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1690028758503,,1690028758999.648e0182e924cbe38699c76e79cee0a5. 2023-07-22T12:27:03,192 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 999a9c03f7b1cabe862a9a76f7912808, disabling compactions & flushes 2023-07-22T12:27:03,192 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1690028766564.999a9c03f7b1cabe862a9a76f7912808. 2023-07-22T12:27:03,192 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1690028766564.999a9c03f7b1cabe862a9a76f7912808. 2023-07-22T12:27:03,192 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1690028766564.999a9c03f7b1cabe862a9a76f7912808. after waiting 0 ms 2023-07-22T12:27:03,192 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1690028766564.999a9c03f7b1cabe862a9a76f7912808. 2023-07-22T12:27:03,196 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/data/backup/system_bulk/999a9c03f7b1cabe862a9a76f7912808/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-07-22T12:27:03,196 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:27:03,197 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1690028766564.999a9c03f7b1cabe862a9a76f7912808. 2023-07-22T12:27:03,197 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 999a9c03f7b1cabe862a9a76f7912808: 2023-07-22T12:27:03,197 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1690028766564.999a9c03f7b1cabe862a9a76f7912808. 2023-07-22T12:27:03,198 INFO [RS:0;jenkins-hbase3:45275 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,45275,1690028749134; all regions closed. 2023-07-22T12:27:03,202 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/WALs/jenkins-hbase3.apache.org,45275,1690028749134/jenkins-hbase3.apache.org%2C45275%2C1690028749134.meta.1690028812319.meta not finished, retry = 0 2023-07-22T12:27:03,308 DEBUG [RS:0;jenkins-hbase3:45275 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/oldWALs 2023-07-22T12:27:03,308 INFO [RS:0;jenkins-hbase3:45275 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C45275%2C1690028749134.meta:.meta(num 1690028812319) 2023-07-22T12:27:03,315 DEBUG [RS:0;jenkins-hbase3:45275 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/oldWALs 2023-07-22T12:27:03,315 INFO [RS:0;jenkins-hbase3:45275 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C45275%2C1690028749134:(num 1690028812337) 2023-07-22T12:27:03,315 DEBUG [RS:0;jenkins-hbase3:45275 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:27:03,315 INFO [RS:0;jenkins-hbase3:45275 {}] regionserver.LeaseManager(133): Closed leases 2023-07-22T12:27:03,315 INFO [RS:0;jenkins-hbase3:45275 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2023-07-22T12:27:03,316 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-07-22T12:27:03,317 INFO [RS:0;jenkins-hbase3:45275 {}] ipc.NettyRpcServer(213): Stopping server on /172.31.12.81:45275 2023-07-22T12:27:03,327 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,45275,1690028749134] 2023-07-22T12:27:03,327 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,45275,1690028749134; numProcessing=1 2023-07-22T12:27:03,329 INFO [RegionServerTracker-0 {}] master.ServerManager(569): Cluster shutdown set; jenkins-hbase3.apache.org,45275,1690028749134 expired; onlineServers=0 2023-07-22T12:27:03,329 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2513): ***** STOPPING region server 'jenkins-hbase3.apache.org,34453,1690028746140' ***** 2023-07-22T12:27:03,329 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2527): STOPPED: Cluster shutdown set; onlineServer=0 2023-07-22T12:27:03,330 DEBUG [M:0;jenkins-hbase3:34453 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13bfa4d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-07-22T12:27:03,330 INFO [M:0;jenkins-hbase3:34453 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-07-22T12:27:03,377 INFO [M:0;jenkins-hbase3:34453 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@4e64b461{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-07-22T12:27:03,378 INFO [M:0;jenkins-hbase3:34453 {}] server.AbstractConnector(383): Stopped ServerConnector@23b09d37{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-07-22T12:27:03,378 INFO [M:0;jenkins-hbase3:34453 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-07-22T12:27:03,378 INFO [M:0;jenkins-hbase3:34453 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@1ef78025{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-07-22T12:27:03,379 INFO [M:0;jenkins-hbase3:34453 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@7c4608eb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/hadoop.log.dir/,STOPPED} 2023-07-22T12:27:03,380 INFO [M:0;jenkins-hbase3:34453 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,34453,1690028746140 2023-07-22T12:27:03,380 INFO [M:0;jenkins-hbase3:34453 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,34453,1690028746140; all regions closed. 2023-07-22T12:27:03,380 DEBUG [M:0;jenkins-hbase3:34453 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:27:03,380 INFO [M:0;jenkins-hbase3:34453 {}] master.HMaster(1664): Stopping master jetty server 2023-07-22T12:27:03,380 INFO [M:0;jenkins-hbase3:34453 {}] server.AbstractConnector(383): Stopped ServerConnector@76516e23{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-07-22T12:27:03,381 DEBUG [M:0;jenkins-hbase3:34453 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-07-22T12:27:03,381 INFO [M:0;jenkins-hbase3:34453 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-07-22T12:27:03,381 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-07-22T12:27:03,381 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1690028751277 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1690028751277,5,FailOnTimeoutGroup] 2023-07-22T12:27:03,381 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1690028751278 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1690028751278,5,FailOnTimeoutGroup] 2023-07-22T12:27:03,381 DEBUG [M:0;jenkins-hbase3:34453 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:27:03,381 DEBUG [M:0;jenkins-hbase3:34453 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-07-22T12:27:03,381 INFO [M:0;jenkins-hbase3:34453 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-07-22T12:27:03,381 DEBUG [M:0;jenkins-hbase3:34453 {}] master.HMaster(1687): Stopping service threads 2023-07-22T12:27:03,382 INFO [M:0;jenkins-hbase3:34453 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-07-22T12:27:03,382 ERROR [M:0;jenkins-hbase3:34453 {}] procedure2.ProcedureExecutor(655): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] 2023-07-22T12:27:03,382 INFO [M:0;jenkins-hbase3:34453 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-07-22T12:27:03,383 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-07-22T12:27:03,383 WARN [M:0;jenkins-hbase3:34453 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-07-22T12:27:03,383 INFO [M:0;jenkins-hbase3:34453 {}] assignment.AssignmentManager(382): Stopping assignment manager 2023-07-22T12:27:03,383 INFO [M:0;jenkins-hbase3:34453 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-07-22T12:27:03,384 DEBUG [M:0;jenkins-hbase3:34453 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-07-22T12:27:03,395 INFO [M:0;jenkins-hbase3:34453 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:27:03,395 DEBUG [M:0;jenkins-hbase3:34453 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:27:03,396 DEBUG [M:0;jenkins-hbase3:34453 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-07-22T12:27:03,396 DEBUG [M:0;jenkins-hbase3:34453 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:27:03,396 INFO [M:0;jenkins-hbase3:34453 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=254.60 KB heapSize=306.32 KB 2023-07-22T12:27:03,409 DEBUG [M:0;jenkins-hbase3:34453 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bd31ecd71c5c4d288bf06f96367bdfe6 is 88, key is hbase:meta,,1/info:sn/1690028751753/Put/seqid=0 2023-07-22T12:27:03,416 INFO [M:0;jenkins-hbase3:34453 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=615 (bloomFilter=true), to=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bd31ecd71c5c4d288bf06f96367bdfe6 2023-07-22T12:27:03,424 DEBUG [M:0;jenkins-hbase3:34453 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: bd31ecd71c5c4d288bf06f96367bdfe6 2023-07-22T12:27:03,428 INFO [RS:0;jenkins-hbase3:45275 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,45275,1690028749134; zookeeper connection closed. 2023-07-22T12:27:03,428 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@bf7808a {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@bf7808a 2023-07-22T12:27:03,429 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-07-22T12:27:03,443 DEBUG [M:0;jenkins-hbase3:34453 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/835cd6978509427a9dc4c9581f319a03 is 2093, key is \x00\x00\x00\x00\x00\x00\x00@/proc:d/1690028817818/Put/seqid=0 2023-07-22T12:27:03,448 INFO [M:0;jenkins-hbase3:34453 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=254.01 KB at sequenceid=615 (bloomFilter=true), to=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/835cd6978509427a9dc4c9581f319a03 2023-07-22T12:27:03,454 INFO [M:0;jenkins-hbase3:34453 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 835cd6978509427a9dc4c9581f319a03 2023-07-22T12:27:03,454 DEBUG [M:0;jenkins-hbase3:34453 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 835cd6978509427a9dc4c9581f319a03 2023-07-22T12:27:03,467 DEBUG [M:0;jenkins-hbase3:34453 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0c4792873d974f82b2be3108bca3e69a is 82, key is jenkins-hbase3.apache.org,45275,1690028749134/rs:state/1690028751320/Put/seqid=0 2023-07-22T12:27:03,871 INFO [M:0;jenkins-hbase3:34453 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=615 (bloomFilter=true), to=hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0c4792873d974f82b2be3108bca3e69a 2023-07-22T12:27:03,877 DEBUG [M:0;jenkins-hbase3:34453 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 0c4792873d974f82b2be3108bca3e69a 2023-07-22T12:27:03,878 DEBUG [M:0;jenkins-hbase3:34453 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bd31ecd71c5c4d288bf06f96367bdfe6 as hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bd31ecd71c5c4d288bf06f96367bdfe6 2023-07-22T12:27:03,884 INFO [M:0;jenkins-hbase3:34453 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bd31ecd71c5c4d288bf06f96367bdfe6, entries=8, sequenceid=615, filesize=5.5 K 2023-07-22T12:27:03,885 DEBUG [M:0;jenkins-hbase3:34453 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/835cd6978509427a9dc4c9581f319a03 as hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/835cd6978509427a9dc4c9581f319a03 2023-07-22T12:27:03,890 INFO [M:0;jenkins-hbase3:34453 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 835cd6978509427a9dc4c9581f319a03 2023-07-22T12:27:03,891 INFO [M:0;jenkins-hbase3:34453 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/835cd6978509427a9dc4c9581f319a03, entries=66, sequenceid=615, filesize=22.2 K 2023-07-22T12:27:03,892 DEBUG [M:0;jenkins-hbase3:34453 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0c4792873d974f82b2be3108bca3e69a as hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0c4792873d974f82b2be3108bca3e69a 2023-07-22T12:27:03,897 INFO [M:0;jenkins-hbase3:34453 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:35695/user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0c4792873d974f82b2be3108bca3e69a, entries=1, sequenceid=615, filesize=5.1 K 2023-07-22T12:27:03,898 INFO [M:0;jenkins-hbase3:34453 {}] regionserver.HRegion(3022): Finished flush of dataSize ~254.60 KB/260714, heapSize ~306.02 KB/313368, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 502ms, sequenceid=615, compaction requested=false 2023-07-22T12:27:03,901 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: bd31ecd71c5c4d288bf06f96367bdfe6 2023-07-22T12:27:03,904 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-proc-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 835cd6978509427a9dc4c9581f319a03 2023-07-22T12:27:03,905 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-rs-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 0c4792873d974f82b2be3108bca3e69a 2023-07-22T12:27:03,906 INFO [M:0;jenkins-hbase3:34453 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:27:03,906 DEBUG [M:0;jenkins-hbase3:34453 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-07-22T12:27:03,910 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/39758229-b0ad-c868-59f3-f76521af36cb/MasterData/WALs/jenkins-hbase3.apache.org,34453,1690028746140/jenkins-hbase3.apache.org%2C34453%2C1690028746140.1690028750404 not finished, retry = 0 2023-07-22T12:27:04,011 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-07-22T12:27:04,011 INFO [M:0;jenkins-hbase3:34453 {}] flush.MasterFlushTableProcedureManager(81): stop: server shutting down. 2023-07-22T12:27:04,011 INFO [M:0;jenkins-hbase3:34453 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-07-22T12:27:04,013 INFO [M:0;jenkins-hbase3:34453 {}] ipc.NettyRpcServer(213): Stopping server on /172.31.12.81:34453 2023-07-22T12:27:04,118 INFO [M:0;jenkins-hbase3:34453 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,34453,1690028746140; zookeeper connection closed. 2023-07-22T12:27:04,121 WARN [Listener at jenkins-hbase3.apache.org/43519 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-07-22T12:27:04,126 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-07-22T12:27:04,236 WARN [BP-1668548271-172.31.12.81-1690028742924 heartbeating to localhost/127.0.0.1:35695 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-07-22T12:27:04,236 WARN [BP-1668548271-172.31.12.81-1690028742924 heartbeating to localhost/127.0.0.1:35695 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-1668548271-172.31.12.81-1690028742924 (Datanode Uuid de9f297b-911d-450f-ae2b-e21372ec3fcc) service to localhost/127.0.0.1:35695 2023-07-22T12:27:04,238 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/cluster_5c7a65c4-cab7-9c85-3939-092136c4beb7/dfs/data/data1/current/BP-1668548271-172.31.12.81-1690028742924 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-07-22T12:27:04,239 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/310080b7-0d87-c19e-6bcf-04a667db9e9c/cluster_5c7a65c4-cab7-9c85-3939-092136c4beb7/dfs/data/data2/current/BP-1668548271-172.31.12.81-1690028742924 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-07-22T12:27:04,282 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-07-22T12:27:04,436 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-07-22T12:27:04,436 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-07-22T12:27:04,447 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-07-22T12:27:04,568 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-07-22T12:27:04,680 ERROR [Thread[Thread-275,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-07-22T12:27:04,680 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-07-22T12:27:04,790 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-07-22T12:27:04,800 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-07-22T12:27:04,802 ERROR [Thread[Thread-285,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-07-22T12:27:04,807 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-07-22T12:27:04,910 ERROR [Thread[Thread-250,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-07-22T12:27:04,910 INFO [Listener at jenkins-hbase3.apache.org/43519 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped