2023-08-10 06:22:30,836 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@776aec5c 2023-08-10 06:22:30,849 main DEBUG Took 0.010347 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-08-10 06:22:30,849 main DEBUG PluginManager 'Core' found 129 plugins 2023-08-10 06:22:30,849 main DEBUG PluginManager 'Level' found 0 plugins 2023-08-10 06:22:30,850 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-08-10 06:22:30,851 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:22:30,859 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-08-10 06:22:30,871 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:22:30,873 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:22:30,874 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:22:30,874 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:22:30,874 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:22:30,874 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:22:30,875 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:22:30,875 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:22:30,876 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:22:30,876 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:22:30,877 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:22:30,877 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:22:30,877 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:22:30,877 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:22:30,878 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:22:30,878 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:22:30,879 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:22:30,879 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:22:30,879 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:22:30,879 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:22:30,880 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:22:30,880 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:22:30,880 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:22:30,880 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:22:30,881 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:22:30,881 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-08-10 06:22:30,883 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:22:30,885 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-08-10 06:22:30,887 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-08-10 06:22:30,887 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-08-10 06:22:30,888 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-08-10 06:22:30,888 main DEBUG PluginManager 'Converter' found 47 plugins 2023-08-10 06:22:30,900 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-08-10 06:22:30,902 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-08-10 06:22:30,904 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-08-10 06:22:30,904 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-08-10 06:22:30,904 main DEBUG createAppenders(={Console}) 2023-08-10 06:22:30,905 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@776aec5c initialized 2023-08-10 06:22:30,905 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@776aec5c 2023-08-10 06:22:30,905 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@776aec5c OK. 2023-08-10 06:22:30,906 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-08-10 06:22:30,906 main DEBUG OutputStream closed 2023-08-10 06:22:30,906 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-08-10 06:22:30,907 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-08-10 06:22:30,907 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@3daa422a OK 2023-08-10 06:22:30,949 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-08-10 06:22:30,951 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-08-10 06:22:30,952 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-08-10 06:22:30,953 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-08-10 06:22:30,954 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-08-10 06:22:30,954 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-08-10 06:22:30,955 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-08-10 06:22:30,955 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-08-10 06:22:30,955 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-08-10 06:22:30,955 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-08-10 06:22:30,956 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-08-10 06:22:30,956 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-08-10 06:22:30,956 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-08-10 06:22:30,956 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-08-10 06:22:30,957 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-08-10 06:22:30,957 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-08-10 06:22:30,957 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-08-10 06:22:30,958 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-08-10 06:22:30,961 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-08-10 06:22:30,961 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@15bbf42f) with optional ClassLoader: null 2023-08-10 06:22:30,961 main DEBUG Shutdown hook enabled. Registering a new one. 2023-08-10 06:22:30,962 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@15bbf42f] started OK. 2023-08-10T06:22:30,992 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures timeout: 13 mins 2023-08-10 06:22:30,996 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-08-10 06:22:30,997 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-08-10T06:22:31,352 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848 2023-08-10T06:22:31,357 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-08-10T06:22:31,359 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:22:31,360 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-08-10T06:22:31,364 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/cluster_9839947b-fd0a-a12a-305c-b4c05ada1651, deleteOnExit=true 2023-08-10T06:22:31,364 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-08-10T06:22:31,365 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/test.cache.data in system properties and HBase conf 2023-08-10T06:22:31,366 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/hadoop.tmp.dir in system properties and HBase conf 2023-08-10T06:22:31,366 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/hadoop.log.dir in system properties and HBase conf 2023-08-10T06:22:31,366 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/mapreduce.cluster.local.dir in system properties and HBase conf 2023-08-10T06:22:31,367 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-08-10T06:22:31,367 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-08-10T06:22:31,463 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-08-10T06:22:31,885 DEBUG [Time-limited test {}] fs.HFileSystem(308): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-08-10T06:22:31,889 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-08-10T06:22:31,890 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-08-10T06:22:31,890 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-08-10T06:22:31,890 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-08-10T06:22:31,891 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-08-10T06:22:31,891 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-08-10T06:22:31,891 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-08-10T06:22:31,892 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/dfs.journalnode.edits.dir in system properties and HBase conf 2023-08-10T06:22:31,892 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-08-10T06:22:31,892 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/nfs.dump.dir in system properties and HBase conf 2023-08-10T06:22:31,893 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/java.io.tmpdir in system properties and HBase conf 2023-08-10T06:22:31,893 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/dfs.journalnode.edits.dir in system properties and HBase conf 2023-08-10T06:22:31,893 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-08-10T06:22:31,893 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-08-10T06:22:32,345 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-08-10T06:22:32,352 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-08-10T06:22:32,646 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-08-10T06:22:32,781 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@19b5229f via org.mortbay.log.Slf4jLog 2023-08-10T06:22:32,798 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-08-10T06:22:32,840 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-08-10T06:22:32,878 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/java.io.tmpdir/Jetty_localhost_45167_hdfs____9ux7az/webapp 2023-08-10T06:22:33,020 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:45167 2023-08-10T06:22:33,048 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-08-10T06:22:33,049 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-08-10T06:22:33,629 WARN [Listener at localhost/46823 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-08-10T06:22:33,695 WARN [Listener at localhost/46823 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-08-10T06:22:33,713 WARN [Listener at localhost/46823 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-08-10T06:22:33,718 INFO [Listener at localhost/46823 {}] log.Slf4jLog(67): jetty-6.1.26 2023-08-10T06:22:33,724 INFO [Listener at localhost/46823 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/java.io.tmpdir/Jetty_localhost_45455_datanode____4l20wq/webapp 2023-08-10T06:22:33,822 INFO [Listener at localhost/46823 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:45455 2023-08-10T06:22:34,143 WARN [Listener at localhost/39747 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-08-10T06:22:34,444 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-08-10T06:22:34,615 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x37b9fc35a4c957be: Processing first storage report for DS-cb72d2f9-6999-4dc2-81ce-221dee4d930e from datanode 52dea3b8-2a59-4572-b9e0-ede2c541b8c9 2023-08-10T06:22:34,617 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x37b9fc35a4c957be: from storage DS-cb72d2f9-6999-4dc2-81ce-221dee4d930e node DatanodeRegistration(127.0.0.1:45175, datanodeUuid=52dea3b8-2a59-4572-b9e0-ede2c541b8c9, infoPort=36917, infoSecurePort=0, ipcPort=39747, storageInfo=lv=-57;cid=testClusterID;nsid=166688944;c=1691648552430), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2023-08-10T06:22:34,617 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x37b9fc35a4c957be: Processing first storage report for DS-6f69260a-0dfd-4736-83b9-45d8e84ba337 from datanode 52dea3b8-2a59-4572-b9e0-ede2c541b8c9 2023-08-10T06:22:34,617 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x37b9fc35a4c957be: from storage DS-6f69260a-0dfd-4736-83b9-45d8e84ba337 node DatanodeRegistration(127.0.0.1:45175, datanodeUuid=52dea3b8-2a59-4572-b9e0-ede2c541b8c9, infoPort=36917, infoSecurePort=0, ipcPort=39747, storageInfo=lv=-57;cid=testClusterID;nsid=166688944;c=1691648552430), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-08-10T06:22:34,687 DEBUG [Listener at localhost/39747 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848 2023-08-10T06:22:34,771 INFO [Listener at localhost/39747 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/cluster_9839947b-fd0a-a12a-305c-b4c05ada1651/zookeeper_0, clientPort=61601, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/cluster_9839947b-fd0a-a12a-305c-b4c05ada1651/zookeeper_0/version-2, dataDirSize=424 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/cluster_9839947b-fd0a-a12a-305c-b4c05ada1651/zookeeper_0/version-2, dataLogSize=424 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, serverId=0 2023-08-10T06:22:34,790 INFO [Listener at localhost/39747 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=61601 2023-08-10T06:22:34,798 INFO [Listener at localhost/39747 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:22:34,800 INFO [Listener at localhost/39747 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:22:35,464 INFO [Listener at localhost/39747 {}] util.FSUtils(461): Created version file at hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322 with version=8 2023-08-10T06:22:35,464 INFO [Listener at localhost/39747 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/hbase-staging 2023-08-10T06:22:35,481 DEBUG [Listener at localhost/39747 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-08-10T06:22:35,481 DEBUG [Listener at localhost/39747 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-08-10T06:22:35,481 DEBUG [Listener at localhost/39747 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-08-10T06:22:35,481 DEBUG [Listener at localhost/39747 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-08-10T06:22:35,695 DEBUG [Listener at localhost/39747 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-08-10T06:22:35,869 INFO [Listener at localhost/39747 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-08-10T06:22:36,527 INFO [Listener at localhost/39747 {}] client.ConnectionUtils(128): master/jenkins-hbase3:0 server-side Connection retries=3 2023-08-10T06:22:36,582 INFO [Listener at localhost/39747 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-08-10T06:22:36,583 INFO [Listener at localhost/39747 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-08-10T06:22:36,583 INFO [Listener at localhost/39747 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-08-10T06:22:36,583 INFO [Listener at localhost/39747 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-08-10T06:22:36,583 INFO [Listener at localhost/39747 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-08-10T06:22:36,736 INFO [Listener at localhost/39747 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-08-10T06:22:36,841 DEBUG [Listener at localhost/39747 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-08-10T06:22:36,850 INFO [Listener at localhost/39747 {}] ipc.NettyRpcServer(177): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-08-10T06:22:36,905 DEBUG [Listener at localhost/39747 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 24310 (auto-detected) 2023-08-10T06:22:36,907 DEBUG [Listener at localhost/39747 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:1c:ff:fe:e2:13:ea (auto-detected) 2023-08-10T06:22:36,945 INFO [Listener at localhost/39747 {}] ipc.NettyRpcServer(144): Bind to /172.31.12.81:39523 2023-08-10T06:22:36,959 INFO [Listener at localhost/39747 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:22:36,961 INFO [Listener at localhost/39747 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:22:36,982 INFO [Listener at localhost/39747 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:39523 connecting to ZooKeeper ensemble=127.0.0.1:61601 2023-08-10T06:22:37,033 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:395230x0, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-08-10T06:22:37,038 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:39523-0x10060e5d75b0000 connected 2023-08-10T06:22:37,061 DEBUG [Listener at localhost/39747 {}] zookeeper.ZKUtil(113): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-08-10T06:22:37,064 DEBUG [Listener at localhost/39747 {}] zookeeper.ZKUtil(113): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-08-10T06:22:37,068 DEBUG [Listener at localhost/39747 {}] zookeeper.ZKUtil(113): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-08-10T06:22:37,079 DEBUG [Listener at localhost/39747 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39523 2023-08-10T06:22:37,079 DEBUG [Listener at localhost/39747 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39523 2023-08-10T06:22:37,079 DEBUG [Listener at localhost/39747 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39523 2023-08-10T06:22:37,080 DEBUG [Listener at localhost/39747 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39523 2023-08-10T06:22:37,080 DEBUG [Listener at localhost/39747 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39523 2023-08-10T06:22:37,134 INFO [Listener at localhost/39747 {}] log.Log(170): Logging initialized @6932ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-08-10T06:22:37,263 INFO [Listener at localhost/39747 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-08-10T06:22:37,264 INFO [Listener at localhost/39747 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-08-10T06:22:37,265 INFO [Listener at localhost/39747 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-08-10T06:22:37,267 INFO [Listener at localhost/39747 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-08-10T06:22:37,267 INFO [Listener at localhost/39747 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-08-10T06:22:37,268 INFO [Listener at localhost/39747 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-08-10T06:22:37,271 INFO [Listener at localhost/39747 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-08-10T06:22:37,402 INFO [Listener at localhost/39747 {}] http.HttpServer(1219): Jetty bound to port 38947 2023-08-10T06:22:37,404 INFO [Listener at localhost/39747 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-08-10T06:22:37,435 INFO [Listener at localhost/39747 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-08-10T06:22:37,439 INFO [Listener at localhost/39747 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@6e84de57{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/hadoop.log.dir/,AVAILABLE} 2023-08-10T06:22:37,440 INFO [Listener at localhost/39747 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-08-10T06:22:37,440 INFO [Listener at localhost/39747 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@44f6082b{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-08-10T06:22:37,503 INFO [Listener at localhost/39747 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-08-10T06:22:37,516 INFO [Listener at localhost/39747 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-08-10T06:22:37,516 INFO [Listener at localhost/39747 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-08-10T06:22:37,518 INFO [Listener at localhost/39747 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-08-10T06:22:37,526 INFO [Listener at localhost/39747 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-08-10T06:22:38,650 INFO [Listener at localhost/39747 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@255e5c0e{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-08-10T06:22:38,664 INFO [Listener at localhost/39747 {}] server.AbstractConnector(333): Started ServerConnector@39508194{HTTP/1.1, (http/1.1)}{0.0.0.0:38947} 2023-08-10T06:22:38,664 INFO [Listener at localhost/39747 {}] server.Server(415): Started @8462ms 2023-08-10T06:22:38,668 INFO [Listener at localhost/39747 {}] master.HMaster(484): hbase.rootdir=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322, hbase.cluster.distributed=false 2023-08-10T06:22:38,733 INFO [Listener at localhost/39747 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase3:0 server-side Connection retries=3 2023-08-10T06:22:38,734 INFO [Listener at localhost/39747 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-08-10T06:22:38,734 INFO [Listener at localhost/39747 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-08-10T06:22:38,734 INFO [Listener at localhost/39747 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-08-10T06:22:38,734 INFO [Listener at localhost/39747 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-08-10T06:22:38,735 INFO [Listener at localhost/39747 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-08-10T06:22:38,745 INFO [Listener at localhost/39747 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-08-10T06:22:38,747 INFO [Listener at localhost/39747 {}] ipc.NettyRpcServer(177): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-08-10T06:22:38,748 INFO [Listener at localhost/39747 {}] ipc.NettyRpcServer(144): Bind to /172.31.12.81:37733 2023-08-10T06:22:38,751 INFO [Listener at localhost/39747 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-08-10T06:22:38,758 DEBUG [Listener at localhost/39747 {}] mob.MobFileCache(120): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-08-10T06:22:38,759 INFO [Listener at localhost/39747 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:22:38,762 INFO [Listener at localhost/39747 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:22:38,764 INFO [Listener at localhost/39747 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:37733 connecting to ZooKeeper ensemble=127.0.0.1:61601 2023-08-10T06:22:38,768 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): regionserver:377330x0, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-08-10T06:22:38,769 DEBUG [Listener at localhost/39747 {}] zookeeper.ZKUtil(113): regionserver:377330x0, quorum=127.0.0.1:61601, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-08-10T06:22:38,769 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:37733-0x10060e5d75b0001 connected 2023-08-10T06:22:38,771 DEBUG [Listener at localhost/39747 {}] zookeeper.ZKUtil(113): regionserver:37733-0x10060e5d75b0001, quorum=127.0.0.1:61601, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-08-10T06:22:38,771 DEBUG [Listener at localhost/39747 {}] zookeeper.ZKUtil(113): regionserver:37733-0x10060e5d75b0001, quorum=127.0.0.1:61601, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-08-10T06:22:38,773 DEBUG [Listener at localhost/39747 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37733 2023-08-10T06:22:38,774 DEBUG [Listener at localhost/39747 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37733 2023-08-10T06:22:38,775 DEBUG [Listener at localhost/39747 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37733 2023-08-10T06:22:38,778 DEBUG [Listener at localhost/39747 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37733 2023-08-10T06:22:38,778 DEBUG [Listener at localhost/39747 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37733 2023-08-10T06:22:38,783 INFO [Listener at localhost/39747 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-08-10T06:22:38,783 INFO [Listener at localhost/39747 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-08-10T06:22:38,783 INFO [Listener at localhost/39747 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-08-10T06:22:38,784 INFO [Listener at localhost/39747 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-08-10T06:22:38,785 INFO [Listener at localhost/39747 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-08-10T06:22:38,785 INFO [Listener at localhost/39747 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-08-10T06:22:38,785 INFO [Listener at localhost/39747 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-08-10T06:22:38,787 INFO [Listener at localhost/39747 {}] http.HttpServer(1219): Jetty bound to port 38113 2023-08-10T06:22:38,787 INFO [Listener at localhost/39747 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-08-10T06:22:38,789 INFO [Listener at localhost/39747 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-08-10T06:22:38,789 INFO [Listener at localhost/39747 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@14e73191{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/hadoop.log.dir/,AVAILABLE} 2023-08-10T06:22:38,789 INFO [Listener at localhost/39747 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-08-10T06:22:38,790 INFO [Listener at localhost/39747 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@53f1d32b{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-08-10T06:22:38,801 INFO [Listener at localhost/39747 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-08-10T06:22:38,802 INFO [Listener at localhost/39747 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-08-10T06:22:38,803 INFO [Listener at localhost/39747 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-08-10T06:22:38,803 INFO [Listener at localhost/39747 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-08-10T06:22:38,804 INFO [Listener at localhost/39747 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-08-10T06:22:38,809 INFO [Listener at localhost/39747 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@578183c5{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-08-10T06:22:38,810 INFO [Listener at localhost/39747 {}] server.AbstractConnector(333): Started ServerConnector@52b0ba37{HTTP/1.1, (http/1.1)}{0.0.0.0:38113} 2023-08-10T06:22:38,810 INFO [Listener at localhost/39747 {}] server.Server(415): Started @8608ms 2023-08-10T06:22:38,817 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-08-10T06:22:38,823 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@2edbc0d5{HTTP/1.1, (http/1.1)}{0.0.0.0:38703} 2023-08-10T06:22:38,823 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @8621ms 2023-08-10T06:22:38,823 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2387): Adding backup master ZNode /1/backup-masters/jenkins-hbase3.apache.org,39523,1691648555675 2023-08-10T06:22:38,835 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37733-0x10060e5d75b0001, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-08-10T06:22:38,835 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-08-10T06:22:38,837 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase3.apache.org,39523,1691648555675 2023-08-10T06:22:38,841 DEBUG [M:0;jenkins-hbase3:39523 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:39523 2023-08-10T06:22:38,856 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-08-10T06:22:38,856 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37733-0x10060e5d75b0001, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-08-10T06:22:38,856 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:22:38,856 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37733-0x10060e5d75b0001, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:22:38,857 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Set watcher on existing znode=/1/master 2023-08-10T06:22:38,859 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Set watcher on existing znode=/1/master 2023-08-10T06:22:38,859 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase3.apache.org,39523,1691648555675 from backup master directory 2023-08-10T06:22:38,864 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase3.apache.org,39523,1691648555675 2023-08-10T06:22:38,864 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37733-0x10060e5d75b0001, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-08-10T06:22:38,865 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-08-10T06:22:38,865 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-08-10T06:22:38,866 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,39523,1691648555675 2023-08-10T06:22:38,869 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-08-10T06:22:38,872 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-08-10T06:22:38,959 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(610): Created cluster ID file at hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/hbase.id with ID: 2fe38447-b08d-4b24-9a29-67ad441f3b56 2023-08-10T06:22:39,000 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:22:39,015 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:22:39,015 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37733-0x10060e5d75b0001, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:22:39,063 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:22:39,065 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-08-10T06:22:39,117 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(270): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-08-10T06:22:39,117 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(276): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-08-10T06:22:39,119 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(285): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:283) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:310) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:950) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:573) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:570) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-08-10T06:22:39,129 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:950) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:573) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:570) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-08-10T06:22:39,131 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-08-10T06:22:39,204 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/MasterData/data/master/store 2023-08-10T06:22:39,228 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-08-10T06:22:39,228 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:22:39,228 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-08-10T06:22:39,229 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:22:39,229 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:22:39,229 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-08-10T06:22:39,229 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:22:39,229 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:22:39,229 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-08-10T06:22:39,231 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/MasterData/data/master/store/.initializing 2023-08-10T06:22:39,231 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/MasterData/WALs/jenkins-hbase3.apache.org,39523,1691648555675 2023-08-10T06:22:39,241 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-08-10T06:22:39,255 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C39523%2C1691648555675, suffix=, logDir=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/MasterData/WALs/jenkins-hbase3.apache.org,39523,1691648555675, archiveDir=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/MasterData/oldWALs, maxLogs=10 2023-08-10T06:22:39,294 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/MasterData/WALs/jenkins-hbase3.apache.org,39523,1691648555675/jenkins-hbase3.apache.org%2C39523%2C1691648555675.1691648559262, exclude list is [], retry=0 2023-08-10T06:22:39,324 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45175,DS-cb72d2f9-6999-4dc2-81ce-221dee4d930e,DISK] 2023-08-10T06:22:39,331 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:346) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:430) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:424) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:482) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:477) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:489) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-08-10T06:22:39,405 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/MasterData/WALs/jenkins-hbase3.apache.org,39523,1691648555675/jenkins-hbase3.apache.org%2C39523%2C1691648555675.1691648559262 2023-08-10T06:22:39,406 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:45175,DS-cb72d2f9-6999-4dc2-81ce-221dee4d930e,DISK]] 2023-08-10T06:22:39,407 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:22:39,407 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:22:39,411 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:22:39,412 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:22:39,501 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:22:39,545 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-08-10T06:22:39,551 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:22:39,555 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:22:39,556 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:22:39,558 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-08-10T06:22:39,559 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:22:39,560 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:22:39,561 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:22:39,564 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-08-10T06:22:39,564 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:22:39,565 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:22:39,565 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:22:39,568 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-08-10T06:22:39,569 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:22:39,570 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:22:39,575 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:22:39,576 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:22:39,592 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-08-10T06:22:39,596 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:22:39,601 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:22:39,602 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11790797600, jitterRate=0.0981035977602005}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-08-10T06:22:39,608 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-08-10T06:22:39,609 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-08-10T06:22:39,646 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@97fb47e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:22:39,703 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(859): No meta location available on zookeeper, skip migrating... 2023-08-10T06:22:39,725 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-08-10T06:22:39,725 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(564): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-08-10T06:22:39,728 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-08-10T06:22:39,730 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(584): Recovered RegionProcedureStore lease in 1 msec 2023-08-10T06:22:39,737 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(598): Loaded RegionProcedureStore in 6 msec 2023-08-10T06:22:39,737 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-08-10T06:22:39,782 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-08-10T06:22:39,806 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-08-10T06:22:39,808 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-08-10T06:22:39,812 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(150): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-08-10T06:22:39,813 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-08-10T06:22:39,814 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-08-10T06:22:39,817 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-08-10T06:22:39,826 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-08-10T06:22:39,828 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-08-10T06:22:39,829 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-08-10T06:22:39,832 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-08-10T06:22:39,848 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-08-10T06:22:39,850 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-08-10T06:22:39,856 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-08-10T06:22:39,856 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:22:39,856 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37733-0x10060e5d75b0001, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-08-10T06:22:39,856 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37733-0x10060e5d75b0001, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:22:39,856 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(803): Active/primary master=jenkins-hbase3.apache.org,39523,1691648555675, sessionid=0x10060e5d75b0000, setting cluster-up flag (Was=false) 2023-08-10T06:22:39,867 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-08-10T06:22:39,876 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:22:39,876 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37733-0x10060e5d75b0001, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:22:39,882 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-08-10T06:22:39,884 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,39523,1691648555675 2023-08-10T06:22:39,889 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37733-0x10060e5d75b0001, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:22:39,889 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:22:39,896 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-08-10T06:22:39,897 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,39523,1691648555675 2023-08-10T06:22:39,903 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37733-0x10060e5d75b0001, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:22:39,903 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:22:39,911 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-08-10T06:22:39,912 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,39523,1691648555675 2023-08-10T06:22:39,986 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver loaded, priority=536870911. 2023-08-10T06:22:40,030 DEBUG [RS:0;jenkins-hbase3:37733 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:37733 2023-08-10T06:22:40,031 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(997): ClusterId : 2fe38447-b08d-4b24-9a29-67ad441f3b56 2023-08-10T06:22:40,034 INFO [RS:0;jenkins-hbase3:37733 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-08-10T06:22:40,038 DEBUG [RS:0;jenkins-hbase3:37733 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-08-10T06:22:40,044 DEBUG [RS:0;jenkins-hbase3:37733 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-08-10T06:22:40,045 DEBUG [RS:0;jenkins-hbase3:37733 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-08-10T06:22:40,048 DEBUG [RS:0;jenkins-hbase3:37733 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-08-10T06:22:40,048 DEBUG [RS:0;jenkins-hbase3:37733 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-08-10T06:22:40,052 DEBUG [RS:0;jenkins-hbase3:37733 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-08-10T06:22:40,053 DEBUG [RS:0;jenkins-hbase3:37733 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123b4ec0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:22:40,058 DEBUG [RS:0;jenkins-hbase3:37733 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d1874f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-08-10T06:22:40,063 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-08-10T06:22:40,063 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-08-10T06:22:40,063 DEBUG [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-08-10T06:22:40,067 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(3023): reportForDuty to master=jenkins-hbase3.apache.org,39523,1691648555675 with isa=jenkins-hbase3.apache.org/172.31.12.81:37733, startcode=1691648558732 2023-08-10T06:22:40,090 DEBUG [RS:0;jenkins-hbase3:37733 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-08-10T06:22:40,090 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-08-10T06:22:40,102 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-08-10T06:22:40,110 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-08-10T06:22:40,122 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,39523,1691648555675 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-08-10T06:22:40,128 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-08-10T06:22:40,128 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-08-10T06:22:40,129 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-08-10T06:22:40,129 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-08-10T06:22:40,129 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-08-10T06:22:40,129 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:22:40,129 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-08-10T06:22:40,130 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:22:40,132 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1691648590132 2023-08-10T06:22:40,135 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-08-10T06:22:40,141 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-08-10T06:22:40,142 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-08-10T06:22:40,143 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-08-10T06:22:40,148 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:22:40,148 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-08-10T06:22:40,156 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-08-10T06:22:40,156 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-08-10T06:22:40,157 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-08-10T06:22:40,158 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-08-10T06:22:40,158 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-08-10T06:22:40,162 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-08-10T06:22:40,178 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-08-10T06:22:40,187 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-08-10T06:22:40,188 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-08-10T06:22:40,189 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-08-10T06:22:40,189 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322 2023-08-10T06:22:40,190 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:51759, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-08-10T06:22:40,197 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x743ed22e to 127.0.0.1:61601 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:22:40,213 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39523 {}] ipc.MetricsHBaseServer(144): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3180) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:588) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-08-10T06:22:40,224 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d2030a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:22:40,224 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-08-10T06:22:40,234 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-08-10T06:22:40,237 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-08-10T06:22:40,242 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1691648560240,5,FailOnTimeoutGroup] 2023-08-10T06:22:40,247 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1691648560242,5,FailOnTimeoutGroup] 2023-08-10T06:22:40,247 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-08-10T06:22:40,248 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1634): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-08-10T06:22:40,250 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-08-10T06:22:40,251 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-08-10T06:22:40,252 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:22:40,255 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-08-10T06:22:40,263 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-08-10T06:22:40,264 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:22:40,265 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:22:40,265 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-08-10T06:22:40,268 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-08-10T06:22:40,268 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:22:40,271 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:22:40,272 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-08-10T06:22:40,273 DEBUG [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(3042): Master is not running yet 2023-08-10T06:22:40,273 WARN [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 100 ms and then retrying. 2023-08-10T06:22:40,275 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-08-10T06:22:40,275 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:22:40,276 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:22:40,278 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/hbase/meta/1588230740 2023-08-10T06:22:40,279 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/hbase/meta/1588230740 2023-08-10T06:22:40,282 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-08-10T06:22:40,285 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-08-10T06:22:40,290 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:22:40,291 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11330405760, jitterRate=0.055226266384124756}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-08-10T06:22:40,294 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-08-10T06:22:40,294 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-08-10T06:22:40,294 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-08-10T06:22:40,294 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-08-10T06:22:40,294 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-08-10T06:22:40,294 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-08-10T06:22:40,296 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-08-10T06:22:40,297 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-08-10T06:22:40,301 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-08-10T06:22:40,301 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-08-10T06:22:40,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-08-10T06:22:40,323 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-08-10T06:22:40,327 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-08-10T06:22:40,375 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(3023): reportForDuty to master=jenkins-hbase3.apache.org,39523,1691648555675 with isa=jenkins-hbase3.apache.org/172.31.12.81:37733, startcode=1691648558732 2023-08-10T06:22:40,380 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39523 {}] master.ServerManager(403): Registering regionserver=jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:22:40,392 DEBUG [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322 2023-08-10T06:22:40,392 DEBUG [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:46823 2023-08-10T06:22:40,392 DEBUG [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=38947 2023-08-10T06:22:40,400 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-08-10T06:22:40,400 DEBUG [RS:0;jenkins-hbase3:37733 {}] zookeeper.ZKUtil(111): regionserver:37733-0x10060e5d75b0001, quorum=127.0.0.1:61601, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:22:40,401 WARN [RS:0;jenkins-hbase3:37733 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-08-10T06:22:40,401 INFO [RS:0;jenkins-hbase3:37733 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-08-10T06:22:40,401 DEBUG [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:22:40,404 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,37733,1691648558732] 2023-08-10T06:22:40,428 DEBUG [RS:0;jenkins-hbase3:37733 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-08-10T06:22:40,438 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-08-10T06:22:40,451 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-08-10T06:22:40,456 INFO [RS:0;jenkins-hbase3:37733 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-08-10T06:22:40,456 INFO [RS:0;jenkins-hbase3:37733 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-08-10T06:22:40,457 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-08-10T06:22:40,466 INFO [RS:0;jenkins-hbase3:37733 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-08-10T06:22:40,466 DEBUG [RS:0;jenkins-hbase3:37733 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:22:40,467 DEBUG [RS:0;jenkins-hbase3:37733 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:22:40,467 DEBUG [RS:0;jenkins-hbase3:37733 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:22:40,467 DEBUG [RS:0;jenkins-hbase3:37733 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:22:40,467 DEBUG [RS:0;jenkins-hbase3:37733 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:22:40,467 DEBUG [RS:0;jenkins-hbase3:37733 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-08-10T06:22:40,467 DEBUG [RS:0;jenkins-hbase3:37733 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:22:40,467 DEBUG [RS:0;jenkins-hbase3:37733 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:22:40,467 DEBUG [RS:0;jenkins-hbase3:37733 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:22:40,467 DEBUG [RS:0;jenkins-hbase3:37733 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:22:40,468 DEBUG [RS:0;jenkins-hbase3:37733 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:22:40,468 DEBUG [RS:0;jenkins-hbase3:37733 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-08-10T06:22:40,469 INFO [RS:0;jenkins-hbase3:37733 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-08-10T06:22:40,469 INFO [RS:0;jenkins-hbase3:37733 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-08-10T06:22:40,469 INFO [RS:0;jenkins-hbase3:37733 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-08-10T06:22:40,469 INFO [RS:0;jenkins-hbase3:37733 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-08-10T06:22:40,469 INFO [RS:0;jenkins-hbase3:37733 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37733,1691648558732-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-08-10T06:22:40,478 WARN [jenkins-hbase3:39523 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-08-10T06:22:40,485 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-08-10T06:22:40,489 INFO [RS:0;jenkins-hbase3:37733 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37733,1691648558732-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-08-10T06:22:40,506 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,37733,1691648558732 started 2023-08-10T06:22:40,506 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,37733,1691648558732, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:37733, sessionid=0x10060e5d75b0001 2023-08-10T06:22:40,506 DEBUG [RS:0;jenkins-hbase3:37733 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-08-10T06:22:40,506 DEBUG [RS:0;jenkins-hbase3:37733 {}] flush.RegionServerFlushTableProcedureManager(106): Start region server flush procedure manager jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:22:40,506 DEBUG [RS:0;jenkins-hbase3:37733 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,37733,1691648558732' 2023-08-10T06:22:40,506 DEBUG [RS:0;jenkins-hbase3:37733 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-08-10T06:22:40,507 DEBUG [RS:0;jenkins-hbase3:37733 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-08-10T06:22:40,508 DEBUG [RS:0;jenkins-hbase3:37733 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-08-10T06:22:40,508 DEBUG [RS:0;jenkins-hbase3:37733 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-08-10T06:22:40,508 DEBUG [RS:0;jenkins-hbase3:37733 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,37733,1691648558732' 2023-08-10T06:22:40,508 DEBUG [RS:0;jenkins-hbase3:37733 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:22:40,508 DEBUG [RS:0;jenkins-hbase3:37733 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:22:40,509 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-08-10T06:22:40,509 DEBUG [RS:0;jenkins-hbase3:37733 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-08-10T06:22:40,509 DEBUG [RS:0;jenkins-hbase3:37733 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-08-10T06:22:40,509 DEBUG [RS:0;jenkins-hbase3:37733 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:22:40,509 DEBUG [RS:0;jenkins-hbase3:37733 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,37733,1691648558732' 2023-08-10T06:22:40,509 DEBUG [RS:0;jenkins-hbase3:37733 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-08-10T06:22:40,510 DEBUG [RS:0;jenkins-hbase3:37733 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-08-10T06:22:40,510 DEBUG [RS:0;jenkins-hbase3:37733 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-08-10T06:22:40,510 INFO [RS:0;jenkins-hbase3:37733 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-08-10T06:22:40,510 INFO [RS:0;jenkins-hbase3:37733 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-08-10T06:22:40,626 INFO [RS:0;jenkins-hbase3:37733 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-08-10T06:22:40,629 INFO [RS:0;jenkins-hbase3:37733 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C37733%2C1691648558732, suffix=, logDir=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732, archiveDir=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/oldWALs, maxLogs=32 2023-08-10T06:22:40,648 DEBUG [RS:0;jenkins-hbase3:37733 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.1691648560631, exclude list is [], retry=0 2023-08-10T06:22:40,660 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45175,DS-cb72d2f9-6999-4dc2-81ce-221dee4d930e,DISK] 2023-08-10T06:22:40,669 INFO [RS:0;jenkins-hbase3:37733 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.1691648560631 2023-08-10T06:22:40,669 DEBUG [RS:0;jenkins-hbase3:37733 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:45175,DS-cb72d2f9-6999-4dc2-81ce-221dee4d930e,DISK]] 2023-08-10T06:22:40,729 DEBUG [jenkins-hbase3:39523 {}] assignment.AssignmentManager(2303): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-08-10T06:22:40,737 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:22:40,746 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,37733,1691648558732, state=OPENING 2023-08-10T06:22:40,755 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-08-10T06:22:40,761 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37733-0x10060e5d75b0001, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:22:40,761 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:22:40,761 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-08-10T06:22:40,761 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-08-10T06:22:40,765 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,37733,1691648558732}] 2023-08-10T06:22:40,972 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(721): New admin connection to jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:22:40,976 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-08-10T06:22:40,981 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:48274, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-08-10T06:22:40,992 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-08-10T06:22:40,993 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-08-10T06:22:40,994 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-08-10T06:22:40,997 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C37733%2C1691648558732.meta, suffix=.meta, logDir=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732, archiveDir=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/oldWALs, maxLogs=32 2023-08-10T06:22:41,009 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.meta.1691648560998.meta, exclude list is [], retry=0 2023-08-10T06:22:41,013 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45175,DS-cb72d2f9-6999-4dc2-81ce-221dee4d930e,DISK] 2023-08-10T06:22:41,018 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.meta.1691648560998.meta 2023-08-10T06:22:41,019 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:45175,DS-cb72d2f9-6999-4dc2-81ce-221dee4d930e,DISK]] 2023-08-10T06:22:41,019 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:22:41,023 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:22:41,024 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-08-10T06:22:41,040 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-08-10T06:22:41,041 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-08-10T06:22:41,046 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-08-10T06:22:41,046 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:22:41,046 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-08-10T06:22:41,046 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-08-10T06:22:41,050 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-08-10T06:22:41,052 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-08-10T06:22:41,052 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:22:41,052 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:22:41,053 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-08-10T06:22:41,054 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-08-10T06:22:41,054 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:22:41,055 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:22:41,055 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-08-10T06:22:41,056 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-08-10T06:22:41,056 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:22:41,057 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:22:41,059 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/hbase/meta/1588230740 2023-08-10T06:22:41,062 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/hbase/meta/1588230740 2023-08-10T06:22:41,066 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-08-10T06:22:41,069 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-08-10T06:22:41,071 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9567311200, jitterRate=-0.10897471010684967}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-08-10T06:22:41,073 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-08-10T06:22:41,090 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2552): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1691648560959 2023-08-10T06:22:41,114 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2579): Finished post open deploy task for hbase:meta,,1.1588230740 2023-08-10T06:22:41,115 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-08-10T06:22:41,116 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:22:41,118 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,37733,1691648558732, state=OPEN 2023-08-10T06:22:41,121 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37733-0x10060e5d75b0001, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-08-10T06:22:41,121 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-08-10T06:22:41,121 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-08-10T06:22:41,121 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-08-10T06:22:41,126 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=3, resume processing ppid=2 2023-08-10T06:22:41,126 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,37733,1691648558732 in 356 msec 2023-08-10T06:22:41,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=2, resume processing ppid=1 2023-08-10T06:22:41,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 954 msec 2023-08-10T06:22:41,283 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.2880 sec 2023-08-10T06:22:41,284 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1062): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1691648561283, completionTime=-1 2023-08-10T06:22:41,284 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(830): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-08-10T06:22:41,284 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1629): Joining cluster... 2023-08-10T06:22:41,369 DEBUG [hconnection-0x7eadca6b-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:22:41,373 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:48286, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:22:41,393 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1641): Number of RegionServers=1 2023-08-10T06:22:41,393 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1691648621393 2023-08-10T06:22:41,393 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1691648681393 2023-08-10T06:22:41,393 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1648): Joined the cluster in 109 msec 2023-08-10T06:22:41,418 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,39523,1691648555675-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-08-10T06:22:41,418 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,39523,1691648555675-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-08-10T06:22:41,418 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,39523,1691648555675-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-08-10T06:22:41,420 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:39523, period=300000, unit=MILLISECONDS is enabled. 2023-08-10T06:22:41,421 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-08-10T06:22:41,429 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-08-10T06:22:41,442 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-08-10T06:22:41,444 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2367): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-08-10T06:22:41,456 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-08-10T06:22:41,459 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:22:41,461 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:22:41,463 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:22:41,886 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 39c91a27ba70e64360f6a7e105d4cca0, NAME => 'hbase:namespace,,1691648561443.39c91a27ba70e64360f6a7e105d4cca0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322 2023-08-10T06:22:41,897 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1691648561443.39c91a27ba70e64360f6a7e105d4cca0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:22:41,897 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing 39c91a27ba70e64360f6a7e105d4cca0, disabling compactions & flushes 2023-08-10T06:22:41,897 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1691648561443.39c91a27ba70e64360f6a7e105d4cca0. 2023-08-10T06:22:41,897 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1691648561443.39c91a27ba70e64360f6a7e105d4cca0. 2023-08-10T06:22:41,897 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1691648561443.39c91a27ba70e64360f6a7e105d4cca0. after waiting 0 ms 2023-08-10T06:22:41,897 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1691648561443.39c91a27ba70e64360f6a7e105d4cca0. 2023-08-10T06:22:41,897 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1691648561443.39c91a27ba70e64360f6a7e105d4cca0. 2023-08-10T06:22:41,897 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for 39c91a27ba70e64360f6a7e105d4cca0: 2023-08-10T06:22:41,900 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:22:41,916 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1691648561443.39c91a27ba70e64360f6a7e105d4cca0.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1691648561901"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648561901"}]},"ts":"1691648561901"} 2023-08-10T06:22:41,949 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:22:41,951 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:22:41,957 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648561951"}]},"ts":"1691648561951"} 2023-08-10T06:22:41,963 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-08-10T06:22:41,968 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=39c91a27ba70e64360f6a7e105d4cca0, ASSIGN}] 2023-08-10T06:22:41,971 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=39c91a27ba70e64360f6a7e105d4cca0, ASSIGN 2023-08-10T06:22:41,973 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=39c91a27ba70e64360f6a7e105d4cca0, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37733,1691648558732; forceNewPlan=false, retain=false 2023-08-10T06:22:42,124 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=39c91a27ba70e64360f6a7e105d4cca0, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:22:42,128 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 39c91a27ba70e64360f6a7e105d4cca0, server=jenkins-hbase3.apache.org,37733,1691648558732}] 2023-08-10T06:22:42,290 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1691648561443.39c91a27ba70e64360f6a7e105d4cca0. 2023-08-10T06:22:42,290 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => 39c91a27ba70e64360f6a7e105d4cca0, NAME => 'hbase:namespace,,1691648561443.39c91a27ba70e64360f6a7e105d4cca0.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:22:42,291 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:22:42,292 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 39c91a27ba70e64360f6a7e105d4cca0 2023-08-10T06:22:42,292 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1691648561443.39c91a27ba70e64360f6a7e105d4cca0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:22:42,292 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for 39c91a27ba70e64360f6a7e105d4cca0 2023-08-10T06:22:42,292 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for 39c91a27ba70e64360f6a7e105d4cca0 2023-08-10T06:22:42,295 INFO [StoreOpener-39c91a27ba70e64360f6a7e105d4cca0-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 39c91a27ba70e64360f6a7e105d4cca0 2023-08-10T06:22:42,298 INFO [StoreOpener-39c91a27ba70e64360f6a7e105d4cca0-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 39c91a27ba70e64360f6a7e105d4cca0 columnFamilyName info 2023-08-10T06:22:42,298 DEBUG [StoreOpener-39c91a27ba70e64360f6a7e105d4cca0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:22:42,299 INFO [StoreOpener-39c91a27ba70e64360f6a7e105d4cca0-1 {}] regionserver.HStore(324): Store=39c91a27ba70e64360f6a7e105d4cca0/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:22:42,301 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/hbase/namespace/39c91a27ba70e64360f6a7e105d4cca0 2023-08-10T06:22:42,301 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/hbase/namespace/39c91a27ba70e64360f6a7e105d4cca0 2023-08-10T06:22:42,305 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for 39c91a27ba70e64360f6a7e105d4cca0 2023-08-10T06:22:42,309 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/hbase/namespace/39c91a27ba70e64360f6a7e105d4cca0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:22:42,309 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened 39c91a27ba70e64360f6a7e105d4cca0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10127706240, jitterRate=-0.056783854961395264}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:22:42,310 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for 39c91a27ba70e64360f6a7e105d4cca0: 2023-08-10T06:22:42,312 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2552): Post open deploy tasks for hbase:namespace,,1691648561443.39c91a27ba70e64360f6a7e105d4cca0., pid=6, masterSystemTime=1691648562282 2023-08-10T06:22:42,316 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2579): Finished post open deploy task for hbase:namespace,,1691648561443.39c91a27ba70e64360f6a7e105d4cca0. 2023-08-10T06:22:42,316 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1691648561443.39c91a27ba70e64360f6a7e105d4cca0. 2023-08-10T06:22:42,317 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=39c91a27ba70e64360f6a7e105d4cca0, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:22:42,326 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=6, resume processing ppid=5 2023-08-10T06:22:42,327 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 39c91a27ba70e64360f6a7e105d4cca0, server=jenkins-hbase3.apache.org,37733,1691648558732 in 195 msec 2023-08-10T06:22:42,331 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=5, resume processing ppid=4 2023-08-10T06:22:42,331 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=39c91a27ba70e64360f6a7e105d4cca0, ASSIGN in 359 msec 2023-08-10T06:22:42,332 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:22:42,333 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648562332"}]},"ts":"1691648562332"} 2023-08-10T06:22:42,336 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-08-10T06:22:42,339 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:22:42,342 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 894 msec 2023-08-10T06:22:42,362 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-08-10T06:22:42,363 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-08-10T06:22:42,363 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37733-0x10060e5d75b0001, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:22:42,363 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:22:42,416 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-08-10T06:22:42,435 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:22:42,441 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 36 msec 2023-08-10T06:22:42,453 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-08-10T06:22:42,466 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:22:42,472 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 19 msec 2023-08-10T06:22:42,480 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-08-10T06:22:42,483 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-08-10T06:22:42,483 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1192): Master has completed initialization 3.617sec 2023-08-10T06:22:42,485 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-08-10T06:22:42,487 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-08-10T06:22:42,488 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-08-10T06:22:42,489 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-08-10T06:22:42,489 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-08-10T06:22:42,491 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,39523,1691648555675-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-08-10T06:22:42,492 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,39523,1691648555675-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-08-10T06:22:42,505 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1295): Balancer post startup initialization complete, took 0 seconds 2023-08-10T06:22:42,506 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-08-10T06:22:42,572 DEBUG [Listener at localhost/39747 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2fa5ba6a to 127.0.0.1:61601 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:22:42,577 DEBUG [Listener at localhost/39747 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e416d18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:22:42,582 DEBUG [Listener at localhost/39747 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-08-10T06:22:42,583 DEBUG [Listener at localhost/39747 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-08-10T06:22:42,599 DEBUG [hconnection-0x761a7a37-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:22:42,612 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:48290, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:22:42,638 INFO [Listener at localhost/39747 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,39523,1691648555675 2023-08-10T06:22:42,638 INFO [Listener at localhost/39747 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-08-10T06:22:42,638 INFO [Listener at localhost/39747 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/test.cache.data in system properties and HBase conf 2023-08-10T06:22:42,638 INFO [Listener at localhost/39747 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/hadoop.tmp.dir in system properties and HBase conf 2023-08-10T06:22:42,639 INFO [Listener at localhost/39747 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/hadoop.log.dir in system properties and HBase conf 2023-08-10T06:22:42,639 INFO [Listener at localhost/39747 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/mapreduce.cluster.local.dir in system properties and HBase conf 2023-08-10T06:22:42,639 INFO [Listener at localhost/39747 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-08-10T06:22:42,639 INFO [Listener at localhost/39747 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-08-10T06:22:42,639 INFO [Listener at localhost/39747 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-08-10T06:22:42,639 INFO [Listener at localhost/39747 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-08-10T06:22:42,639 INFO [Listener at localhost/39747 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-08-10T06:22:42,639 INFO [Listener at localhost/39747 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-08-10T06:22:42,639 INFO [Listener at localhost/39747 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-08-10T06:22:42,639 INFO [Listener at localhost/39747 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-08-10T06:22:42,640 INFO [Listener at localhost/39747 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-08-10T06:22:42,640 INFO [Listener at localhost/39747 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/dfs.journalnode.edits.dir in system properties and HBase conf 2023-08-10T06:22:42,640 INFO [Listener at localhost/39747 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-08-10T06:22:42,640 INFO [Listener at localhost/39747 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/nfs.dump.dir in system properties and HBase conf 2023-08-10T06:22:42,640 INFO [Listener at localhost/39747 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/java.io.tmpdir in system properties and HBase conf 2023-08-10T06:22:42,640 INFO [Listener at localhost/39747 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/dfs.journalnode.edits.dir in system properties and HBase conf 2023-08-10T06:22:42,640 INFO [Listener at localhost/39747 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-08-10T06:22:42,640 INFO [Listener at localhost/39747 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-08-10T06:22:44,246 WARN [Thread-249 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-08-10T06:22:44,619 INFO [Thread-249 {}] log.Slf4jLog(67): jetty-6.1.26 2023-08-10T06:22:44,625 INFO [Thread-249 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_45069_jobhistory____.z5pso4/webapp 2023-08-10T06:22:44,788 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-08-10T06:22:44,788 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-08-10T06:22:44,788 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-08-10T06:22:44,791 INFO [Thread-249 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-08-10T06:22:44,881 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:22:45,007 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:22:45,355 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-08-10T06:22:45,426 INFO [Thread-249 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:45069 2023-08-10T06:22:45,742 WARN [Listener at jenkins-hbase3.apache.org/33795 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-08-10T06:22:45,824 WARN [Listener at jenkins-hbase3.apache.org/33795 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-08-10T06:22:45,824 INFO [Listener at jenkins-hbase3.apache.org/33795 {}] log.Slf4jLog(67): jetty-6.1.26 2023-08-10T06:22:45,830 INFO [Listener at jenkins-hbase3.apache.org/33795 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_34383_cluster____61zmvq/webapp 2023-08-10T06:22:45,864 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-08-10T06:22:45,865 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-08-10T06:22:45,874 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-08-10T06:22:45,874 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-08-10T06:22:45,876 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: Master,sub=Coprocessor.Master.CP_org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver 2023-08-10T06:22:45,876 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering Master,sub=Coprocessor.Master.CP_org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver Metrics about HBase MasterObservers 2023-08-10T06:22:45,876 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:22:45,876 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-08-10T06:22:45,876 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-08-10T06:22:45,876 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-08-10T06:22:45,877 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-08-10T06:22:45,877 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-08-10T06:22:45,940 INFO [Listener at jenkins-hbase3.apache.org/33795 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-08-10T06:22:45,940 INFO [Listener at jenkins-hbase3.apache.org/33795 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-08-10T06:22:45,940 INFO [Listener at jenkins-hbase3.apache.org/33795 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-08-10T06:22:45,941 INFO [Listener at jenkins-hbase3.apache.org/33795 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-08-10T06:22:45,951 INFO [Listener at jenkins-hbase3.apache.org/33795 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:22:46,159 INFO [Listener at jenkins-hbase3.apache.org/33795 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:22:46,274 INFO [Listener at jenkins-hbase3.apache.org/33795 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:22:46,281 INFO [Listener at jenkins-hbase3.apache.org/33795 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:34383 2023-08-10T06:22:46,589 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:22:46,666 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-08-10T06:22:46,668 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-08-10T06:22:46,748 WARN [Listener at jenkins-hbase3.apache.org/41571 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-08-10T06:22:46,772 WARN [Listener at jenkins-hbase3.apache.org/41571 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-08-10T06:22:46,773 INFO [Listener at jenkins-hbase3.apache.org/41571 {}] log.Slf4jLog(67): jetty-6.1.26 2023-08-10T06:22:46,779 INFO [Listener at jenkins-hbase3.apache.org/41571 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_37249_node____8se06q/webapp 2023-08-10T06:22:46,831 INFO [Listener at jenkins-hbase3.apache.org/41571 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-08-10T06:22:46,831 INFO [Listener at jenkins-hbase3.apache.org/41571 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-08-10T06:22:46,831 INFO [Listener at jenkins-hbase3.apache.org/41571 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-08-10T06:22:46,831 INFO [Listener at jenkins-hbase3.apache.org/41571 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-08-10T06:22:46,839 INFO [Listener at jenkins-hbase3.apache.org/41571 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:22:46,861 INFO [Listener at jenkins-hbase3.apache.org/41571 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:22:46,943 INFO [Listener at jenkins-hbase3.apache.org/41571 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:22:46,962 INFO [Listener at jenkins-hbase3.apache.org/41571 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:37249 2023-08-10T06:22:47,149 WARN [Listener at jenkins-hbase3.apache.org/36469 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-08-10T06:22:47,158 WARN [Listener at jenkins-hbase3.apache.org/36469 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-08-10T06:22:47,159 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] log.Slf4jLog(67): jetty-6.1.26 2023-08-10T06:22:47,168 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_37703_node____.ihckln/webapp 2023-08-10T06:22:47,244 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-08-10T06:22:47,245 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-08-10T06:22:47,245 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-08-10T06:22:47,245 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-08-10T06:22:47,257 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:22:47,283 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:22:47,369 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:22:47,378 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:37703 2023-08-10T06:22:47,409 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-08-10T06:22:47,409 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:46823/backupUT 2023-08-10T06:22:47,421 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:22:47,425 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:48964, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:22:47,434 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns1'} 2023-08-10T06:22:47,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure2.ProcedureExecutor(1033): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-08-10T06:22:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=9 2023-08-10T06:22:47,463 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:22:47,468 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 32 msec 2023-08-10T06:22:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=9 2023-08-10T06:22:47,558 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns2'} 2023-08-10T06:22:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure2.ProcedureExecutor(1033): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-08-10T06:22:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=10 2023-08-10T06:22:47,572 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:22:47,579 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 19 msec 2023-08-10T06:22:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=10 2023-08-10T06:22:47,669 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns3'} 2023-08-10T06:22:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure2.ProcedureExecutor(1033): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-08-10T06:22:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=11 2023-08-10T06:22:47,684 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:22:47,689 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 18 msec 2023-08-10T06:22:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=11 2023-08-10T06:22:47,782 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns4'} 2023-08-10T06:22:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure2.ProcedureExecutor(1033): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-08-10T06:22:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=12 2023-08-10T06:22:47,797 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:22:47,803 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 18 msec 2023-08-10T06:22:47,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=12 2023-08-10T06:22:47,911 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'test-1691648567409', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:22:47,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure2.ProcedureExecutor(1033): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1691648567409 2023-08-10T06:22:47,916 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1691648567409 execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:22:47,917 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:22:47,917 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "test-1691648567409" procId is: 13 2023-08-10T06:22:47,918 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1691648567409 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:22:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-08-10T06:22:47,953 INFO [RegionOpenAndInit-test-1691648567409-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 89ac3e8e753d2c1222fc1445a7518dbe, NAME => 'test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1691648567409', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322 2023-08-10T06:22:48,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-08-10T06:22:48,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-08-10T06:22:48,374 DEBUG [RegionOpenAndInit-test-1691648567409-pool-0 {}] regionserver.HRegion(888): Instantiated test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:22:48,374 DEBUG [RegionOpenAndInit-test-1691648567409-pool-0 {}] regionserver.HRegion(1668): Closing 89ac3e8e753d2c1222fc1445a7518dbe, disabling compactions & flushes 2023-08-10T06:22:48,374 INFO [RegionOpenAndInit-test-1691648567409-pool-0 {}] regionserver.HRegion(1690): Closing region test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe. 2023-08-10T06:22:48,374 DEBUG [RegionOpenAndInit-test-1691648567409-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe. 2023-08-10T06:22:48,374 DEBUG [RegionOpenAndInit-test-1691648567409-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe. after waiting 0 ms 2023-08-10T06:22:48,374 DEBUG [RegionOpenAndInit-test-1691648567409-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe. 2023-08-10T06:22:48,374 INFO [RegionOpenAndInit-test-1691648567409-pool-0 {}] regionserver.HRegion(1909): Closed test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe. 2023-08-10T06:22:48,374 DEBUG [RegionOpenAndInit-test-1691648567409-pool-0 {}] regionserver.HRegion(1622): Region close journal for 89ac3e8e753d2c1222fc1445a7518dbe: 2023-08-10T06:22:48,376 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1691648567409 execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:22:48,376 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1691648568376"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648568376"}]},"ts":"1691648568376"} 2023-08-10T06:22:48,379 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:22:48,380 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1691648567409 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:22:48,381 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1691648567409","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648568380"}]},"ts":"1691648568380"} 2023-08-10T06:22:48,384 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1691648567409, state=ENABLING in hbase:meta 2023-08-10T06:22:48,388 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1691648567409, region=89ac3e8e753d2c1222fc1445a7518dbe, ASSIGN}] 2023-08-10T06:22:48,390 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1691648567409, region=89ac3e8e753d2c1222fc1445a7518dbe, ASSIGN 2023-08-10T06:22:48,392 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1691648567409, region=89ac3e8e753d2c1222fc1445a7518dbe, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37733,1691648558732; forceNewPlan=false, retain=false 2023-08-10T06:22:48,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-08-10T06:22:48,543 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=89ac3e8e753d2c1222fc1445a7518dbe, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:22:48,547 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure 89ac3e8e753d2c1222fc1445a7518dbe, server=jenkins-hbase3.apache.org,37733,1691648558732}] 2023-08-10T06:22:48,717 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe. 2023-08-10T06:22:48,717 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => 89ac3e8e753d2c1222fc1445a7518dbe, NAME => 'test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:22:48,718 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:22:48,718 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1691648567409 89ac3e8e753d2c1222fc1445a7518dbe 2023-08-10T06:22:48,718 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:22:48,718 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for 89ac3e8e753d2c1222fc1445a7518dbe 2023-08-10T06:22:48,718 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for 89ac3e8e753d2c1222fc1445a7518dbe 2023-08-10T06:22:48,721 INFO [StoreOpener-89ac3e8e753d2c1222fc1445a7518dbe-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 89ac3e8e753d2c1222fc1445a7518dbe 2023-08-10T06:22:48,723 INFO [StoreOpener-89ac3e8e753d2c1222fc1445a7518dbe-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89ac3e8e753d2c1222fc1445a7518dbe columnFamilyName f 2023-08-10T06:22:48,724 DEBUG [StoreOpener-89ac3e8e753d2c1222fc1445a7518dbe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:22:48,725 INFO [StoreOpener-89ac3e8e753d2c1222fc1445a7518dbe-1 {}] regionserver.HStore(324): Store=89ac3e8e753d2c1222fc1445a7518dbe/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:22:48,726 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/default/test-1691648567409/89ac3e8e753d2c1222fc1445a7518dbe 2023-08-10T06:22:48,727 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/default/test-1691648567409/89ac3e8e753d2c1222fc1445a7518dbe 2023-08-10T06:22:48,733 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for 89ac3e8e753d2c1222fc1445a7518dbe 2023-08-10T06:22:48,742 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/default/test-1691648567409/89ac3e8e753d2c1222fc1445a7518dbe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:22:48,743 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened 89ac3e8e753d2c1222fc1445a7518dbe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10323192800, jitterRate=-0.03857775032520294}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:22:48,744 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for 89ac3e8e753d2c1222fc1445a7518dbe: 2023-08-10T06:22:48,745 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2552): Post open deploy tasks for test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe., pid=15, masterSystemTime=1691648568700 2023-08-10T06:22:48,748 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2579): Finished post open deploy task for test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe. 2023-08-10T06:22:48,748 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe. 2023-08-10T06:22:48,749 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=89ac3e8e753d2c1222fc1445a7518dbe, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:22:48,755 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=15, resume processing ppid=14 2023-08-10T06:22:48,755 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure 89ac3e8e753d2c1222fc1445a7518dbe, server=jenkins-hbase3.apache.org,37733,1691648558732 in 206 msec 2023-08-10T06:22:48,758 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=14, resume processing ppid=13 2023-08-10T06:22:48,758 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1691648567409, region=89ac3e8e753d2c1222fc1445a7518dbe, ASSIGN in 367 msec 2023-08-10T06:22:48,759 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1691648567409 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:22:48,759 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1691648567409","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648568759"}]},"ts":"1691648568759"} 2023-08-10T06:22:48,761 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1691648567409, state=ENABLED in hbase:meta 2023-08-10T06:22:48,764 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1691648567409 execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:22:48,766 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1691648567409 in 853 msec 2023-08-10T06:22:49,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-08-10T06:22:49,037 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: default:test-1691648567409, procId: 13 completed 2023-08-10T06:22:49,037 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x172202a0 to 127.0.0.1:61601 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:22:49,044 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cbbb3ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:22:49,046 DEBUG [hconnection-0x754e4af1-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:22:49,050 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:48302, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:22:49,056 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37733 {}] regionserver.HRegion(8210): writing data to region test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe. with WAL disabled. Data may be lost in the event of a crash. 2023-08-10T06:22:49,165 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns2:test-16916485674091', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:22:49,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure2.ProcedureExecutor(1033): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-16916485674091 2023-08-10T06:22:49,169 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16916485674091 execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:22:49,169 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:22:49,170 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "test-16916485674091" procId is: 16 2023-08-10T06:22:49,171 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-16916485674091 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:22:49,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-08-10T06:22:49,196 INFO [RegionOpenAndInit-ns2:test-16916485674091-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => b203bddb45f38798df364fa4839b7781, NAME => 'ns2:test-16916485674091,,1691648569165.b203bddb45f38798df364fa4839b7781.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16916485674091', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322 2023-08-10T06:22:49,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-08-10T06:22:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-08-10T06:22:49,620 DEBUG [RegionOpenAndInit-ns2:test-16916485674091-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16916485674091,,1691648569165.b203bddb45f38798df364fa4839b7781.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:22:49,621 DEBUG [RegionOpenAndInit-ns2:test-16916485674091-pool-0 {}] regionserver.HRegion(1668): Closing b203bddb45f38798df364fa4839b7781, disabling compactions & flushes 2023-08-10T06:22:49,621 INFO [RegionOpenAndInit-ns2:test-16916485674091-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16916485674091,,1691648569165.b203bddb45f38798df364fa4839b7781. 2023-08-10T06:22:49,621 DEBUG [RegionOpenAndInit-ns2:test-16916485674091-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16916485674091,,1691648569165.b203bddb45f38798df364fa4839b7781. 2023-08-10T06:22:49,621 DEBUG [RegionOpenAndInit-ns2:test-16916485674091-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16916485674091,,1691648569165.b203bddb45f38798df364fa4839b7781. after waiting 0 ms 2023-08-10T06:22:49,621 DEBUG [RegionOpenAndInit-ns2:test-16916485674091-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16916485674091,,1691648569165.b203bddb45f38798df364fa4839b7781. 2023-08-10T06:22:49,621 INFO [RegionOpenAndInit-ns2:test-16916485674091-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16916485674091,,1691648569165.b203bddb45f38798df364fa4839b7781. 2023-08-10T06:22:49,621 DEBUG [RegionOpenAndInit-ns2:test-16916485674091-pool-0 {}] regionserver.HRegion(1622): Region close journal for b203bddb45f38798df364fa4839b7781: 2023-08-10T06:22:49,623 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-16916485674091 execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:22:49,623 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16916485674091,,1691648569165.b203bddb45f38798df364fa4839b7781.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1691648569623"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648569623"}]},"ts":"1691648569623"} 2023-08-10T06:22:49,626 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:22:49,627 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-16916485674091 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:22:49,628 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16916485674091","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648569627"}]},"ts":"1691648569627"} 2023-08-10T06:22:49,630 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16916485674091, state=ENABLING in hbase:meta 2023-08-10T06:22:49,636 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16916485674091, region=b203bddb45f38798df364fa4839b7781, ASSIGN}] 2023-08-10T06:22:49,638 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16916485674091, region=b203bddb45f38798df364fa4839b7781, ASSIGN 2023-08-10T06:22:49,640 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16916485674091, region=b203bddb45f38798df364fa4839b7781, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37733,1691648558732; forceNewPlan=false, retain=false 2023-08-10T06:22:49,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-08-10T06:22:49,792 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=b203bddb45f38798df364fa4839b7781, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:22:49,795 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure b203bddb45f38798df364fa4839b7781, server=jenkins-hbase3.apache.org,37733,1691648558732}] 2023-08-10T06:22:49,953 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-16916485674091,,1691648569165.b203bddb45f38798df364fa4839b7781. 2023-08-10T06:22:49,953 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => b203bddb45f38798df364fa4839b7781, NAME => 'ns2:test-16916485674091,,1691648569165.b203bddb45f38798df364fa4839b7781.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:22:49,954 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:22:49,954 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16916485674091 b203bddb45f38798df364fa4839b7781 2023-08-10T06:22:49,954 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-16916485674091,,1691648569165.b203bddb45f38798df364fa4839b7781.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:22:49,954 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for b203bddb45f38798df364fa4839b7781 2023-08-10T06:22:49,954 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for b203bddb45f38798df364fa4839b7781 2023-08-10T06:22:49,956 INFO [StoreOpener-b203bddb45f38798df364fa4839b7781-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region b203bddb45f38798df364fa4839b7781 2023-08-10T06:22:49,958 INFO [StoreOpener-b203bddb45f38798df364fa4839b7781-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b203bddb45f38798df364fa4839b7781 columnFamilyName f 2023-08-10T06:22:49,958 DEBUG [StoreOpener-b203bddb45f38798df364fa4839b7781-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:22:49,959 INFO [StoreOpener-b203bddb45f38798df364fa4839b7781-1 {}] regionserver.HStore(324): Store=b203bddb45f38798df364fa4839b7781/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:22:49,961 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/ns2/test-16916485674091/b203bddb45f38798df364fa4839b7781 2023-08-10T06:22:49,961 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/ns2/test-16916485674091/b203bddb45f38798df364fa4839b7781 2023-08-10T06:22:49,965 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for b203bddb45f38798df364fa4839b7781 2023-08-10T06:22:49,973 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/ns2/test-16916485674091/b203bddb45f38798df364fa4839b7781/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:22:49,974 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened b203bddb45f38798df364fa4839b7781; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9453480960, jitterRate=-0.11957597732543945}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:22:49,975 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for b203bddb45f38798df364fa4839b7781: 2023-08-10T06:22:49,976 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2552): Post open deploy tasks for ns2:test-16916485674091,,1691648569165.b203bddb45f38798df364fa4839b7781., pid=18, masterSystemTime=1691648569948 2023-08-10T06:22:49,978 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2579): Finished post open deploy task for ns2:test-16916485674091,,1691648569165.b203bddb45f38798df364fa4839b7781. 2023-08-10T06:22:49,978 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-16916485674091,,1691648569165.b203bddb45f38798df364fa4839b7781. 2023-08-10T06:22:49,979 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=b203bddb45f38798df364fa4839b7781, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:22:49,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=18, resume processing ppid=17 2023-08-10T06:22:49,986 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure b203bddb45f38798df364fa4839b7781, server=jenkins-hbase3.apache.org,37733,1691648558732 in 187 msec 2023-08-10T06:22:49,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=17, resume processing ppid=16 2023-08-10T06:22:49,989 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16916485674091, region=b203bddb45f38798df364fa4839b7781, ASSIGN in 350 msec 2023-08-10T06:22:49,990 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-16916485674091 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:22:49,990 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16916485674091","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648569990"}]},"ts":"1691648569990"} 2023-08-10T06:22:49,992 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16916485674091, state=ENABLED in hbase:meta 2023-08-10T06:22:49,995 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16916485674091 execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:22:49,997 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-16916485674091 in 830 msec 2023-08-10T06:22:50,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-08-10T06:22:50,280 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns2:test-16916485674091, procId: 16 completed 2023-08-10T06:22:50,284 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37733 {}] regionserver.HRegion(8210): writing data to region ns2:test-16916485674091,,1691648569165.b203bddb45f38798df364fa4839b7781. with WAL disabled. Data may be lost in the event of a crash. 2023-08-10T06:22:50,371 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns3:test-16916485674092', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:22:50,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure2.ProcedureExecutor(1033): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-16916485674092 2023-08-10T06:22:50,375 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16916485674092 execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:22:50,375 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:22:50,375 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "test-16916485674092" procId is: 19 2023-08-10T06:22:50,376 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-16916485674092 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:22:50,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-08-10T06:22:50,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-08-10T06:22:50,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-08-10T06:22:50,793 INFO [RegionOpenAndInit-ns3:test-16916485674092-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 186ba7be42f319c9f99e9a75814a2785, NAME => 'ns3:test-16916485674092,,1691648570371.186ba7be42f319c9f99e9a75814a2785.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16916485674092', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322 2023-08-10T06:22:50,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-08-10T06:22:51,215 DEBUG [RegionOpenAndInit-ns3:test-16916485674092-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16916485674092,,1691648570371.186ba7be42f319c9f99e9a75814a2785.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:22:51,215 DEBUG [RegionOpenAndInit-ns3:test-16916485674092-pool-0 {}] regionserver.HRegion(1668): Closing 186ba7be42f319c9f99e9a75814a2785, disabling compactions & flushes 2023-08-10T06:22:51,215 INFO [RegionOpenAndInit-ns3:test-16916485674092-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16916485674092,,1691648570371.186ba7be42f319c9f99e9a75814a2785. 2023-08-10T06:22:51,215 DEBUG [RegionOpenAndInit-ns3:test-16916485674092-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16916485674092,,1691648570371.186ba7be42f319c9f99e9a75814a2785. 2023-08-10T06:22:51,215 DEBUG [RegionOpenAndInit-ns3:test-16916485674092-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16916485674092,,1691648570371.186ba7be42f319c9f99e9a75814a2785. after waiting 0 ms 2023-08-10T06:22:51,215 DEBUG [RegionOpenAndInit-ns3:test-16916485674092-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16916485674092,,1691648570371.186ba7be42f319c9f99e9a75814a2785. 2023-08-10T06:22:51,215 INFO [RegionOpenAndInit-ns3:test-16916485674092-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16916485674092,,1691648570371.186ba7be42f319c9f99e9a75814a2785. 2023-08-10T06:22:51,215 DEBUG [RegionOpenAndInit-ns3:test-16916485674092-pool-0 {}] regionserver.HRegion(1622): Region close journal for 186ba7be42f319c9f99e9a75814a2785: 2023-08-10T06:22:51,217 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-16916485674092 execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:22:51,217 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16916485674092,,1691648570371.186ba7be42f319c9f99e9a75814a2785.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1691648571217"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648571217"}]},"ts":"1691648571217"} 2023-08-10T06:22:51,219 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:22:51,220 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-16916485674092 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:22:51,221 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16916485674092","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648571220"}]},"ts":"1691648571220"} 2023-08-10T06:22:51,222 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16916485674092, state=ENABLING in hbase:meta 2023-08-10T06:22:51,226 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16916485674092, region=186ba7be42f319c9f99e9a75814a2785, ASSIGN}] 2023-08-10T06:22:51,228 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16916485674092, region=186ba7be42f319c9f99e9a75814a2785, ASSIGN 2023-08-10T06:22:51,229 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16916485674092, region=186ba7be42f319c9f99e9a75814a2785, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37733,1691648558732; forceNewPlan=false, retain=false 2023-08-10T06:22:51,381 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=186ba7be42f319c9f99e9a75814a2785, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:22:51,384 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure 186ba7be42f319c9f99e9a75814a2785, server=jenkins-hbase3.apache.org,37733,1691648558732}] 2023-08-10T06:22:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-08-10T06:22:51,543 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-16916485674092,,1691648570371.186ba7be42f319c9f99e9a75814a2785. 2023-08-10T06:22:51,543 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => 186ba7be42f319c9f99e9a75814a2785, NAME => 'ns3:test-16916485674092,,1691648570371.186ba7be42f319c9f99e9a75814a2785.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:22:51,544 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:22:51,544 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16916485674092 186ba7be42f319c9f99e9a75814a2785 2023-08-10T06:22:51,544 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-16916485674092,,1691648570371.186ba7be42f319c9f99e9a75814a2785.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:22:51,544 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for 186ba7be42f319c9f99e9a75814a2785 2023-08-10T06:22:51,544 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for 186ba7be42f319c9f99e9a75814a2785 2023-08-10T06:22:51,546 INFO [StoreOpener-186ba7be42f319c9f99e9a75814a2785-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 186ba7be42f319c9f99e9a75814a2785 2023-08-10T06:22:51,548 INFO [StoreOpener-186ba7be42f319c9f99e9a75814a2785-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 186ba7be42f319c9f99e9a75814a2785 columnFamilyName f 2023-08-10T06:22:51,548 DEBUG [StoreOpener-186ba7be42f319c9f99e9a75814a2785-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:22:51,549 INFO [StoreOpener-186ba7be42f319c9f99e9a75814a2785-1 {}] regionserver.HStore(324): Store=186ba7be42f319c9f99e9a75814a2785/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:22:51,549 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/ns3/test-16916485674092/186ba7be42f319c9f99e9a75814a2785 2023-08-10T06:22:51,550 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/ns3/test-16916485674092/186ba7be42f319c9f99e9a75814a2785 2023-08-10T06:22:51,554 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for 186ba7be42f319c9f99e9a75814a2785 2023-08-10T06:22:51,557 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/ns3/test-16916485674092/186ba7be42f319c9f99e9a75814a2785/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:22:51,558 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened 186ba7be42f319c9f99e9a75814a2785; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11021182240, jitterRate=0.026427581906318665}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:22:51,558 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for 186ba7be42f319c9f99e9a75814a2785: 2023-08-10T06:22:51,560 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2552): Post open deploy tasks for ns3:test-16916485674092,,1691648570371.186ba7be42f319c9f99e9a75814a2785., pid=21, masterSystemTime=1691648571538 2023-08-10T06:22:51,562 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2579): Finished post open deploy task for ns3:test-16916485674092,,1691648570371.186ba7be42f319c9f99e9a75814a2785. 2023-08-10T06:22:51,562 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-16916485674092,,1691648570371.186ba7be42f319c9f99e9a75814a2785. 2023-08-10T06:22:51,563 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=186ba7be42f319c9f99e9a75814a2785, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:22:51,567 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=21, resume processing ppid=20 2023-08-10T06:22:51,567 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure 186ba7be42f319c9f99e9a75814a2785, server=jenkins-hbase3.apache.org,37733,1691648558732 in 181 msec 2023-08-10T06:22:51,570 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=20, resume processing ppid=19 2023-08-10T06:22:51,570 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16916485674092, region=186ba7be42f319c9f99e9a75814a2785, ASSIGN in 341 msec 2023-08-10T06:22:51,571 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-16916485674092 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:22:51,571 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16916485674092","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648571571"}]},"ts":"1691648571571"} 2023-08-10T06:22:51,572 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16916485674092, state=ENABLED in hbase:meta 2023-08-10T06:22:51,575 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16916485674092 execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:22:51,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-16916485674092 in 1.2040 sec 2023-08-10T06:22:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-08-10T06:22:52,486 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns3:test-16916485674092, procId: 19 completed 2023-08-10T06:22:52,487 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-16916485674092 get assigned. Timeout = 60000ms 2023-08-10T06:22:52,489 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-08-10T06:22:52,493 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-16916485674092 assigned to meta. Checking AM states. 2023-08-10T06:22:52,494 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-08-10T06:22:52,494 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-16916485674092 assigned. 2023-08-10T06:22:52,496 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns4:test-16916485674093', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:22:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure2.ProcedureExecutor(1033): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-16916485674093 2023-08-10T06:22:52,500 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16916485674093 execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:22:52,500 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:22:52,500 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns4" qualifier: "test-16916485674093" procId is: 22 2023-08-10T06:22:52,501 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-16916485674093 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:22:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-08-10T06:22:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-08-10T06:22:52,669 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-16916485674091' 2023-08-10T06:22:52,670 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1691648567409' 2023-08-10T06:22:52,670 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16916485674092' 2023-08-10T06:22:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-08-10T06:22:52,916 INFO [RegionOpenAndInit-ns4:test-16916485674093-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 47fa27f3f64d1137e3edbb66cac8a6d3, NAME => 'ns4:test-16916485674093,,1691648572496.47fa27f3f64d1137e3edbb66cac8a6d3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-16916485674093', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322 2023-08-10T06:22:53,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-08-10T06:22:53,334 DEBUG [RegionOpenAndInit-ns4:test-16916485674093-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-16916485674093,,1691648572496.47fa27f3f64d1137e3edbb66cac8a6d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:22:53,334 DEBUG [RegionOpenAndInit-ns4:test-16916485674093-pool-0 {}] regionserver.HRegion(1668): Closing 47fa27f3f64d1137e3edbb66cac8a6d3, disabling compactions & flushes 2023-08-10T06:22:53,334 INFO [RegionOpenAndInit-ns4:test-16916485674093-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-16916485674093,,1691648572496.47fa27f3f64d1137e3edbb66cac8a6d3. 2023-08-10T06:22:53,334 DEBUG [RegionOpenAndInit-ns4:test-16916485674093-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16916485674093,,1691648572496.47fa27f3f64d1137e3edbb66cac8a6d3. 2023-08-10T06:22:53,334 DEBUG [RegionOpenAndInit-ns4:test-16916485674093-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16916485674093,,1691648572496.47fa27f3f64d1137e3edbb66cac8a6d3. after waiting 0 ms 2023-08-10T06:22:53,334 DEBUG [RegionOpenAndInit-ns4:test-16916485674093-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16916485674093,,1691648572496.47fa27f3f64d1137e3edbb66cac8a6d3. 2023-08-10T06:22:53,334 INFO [RegionOpenAndInit-ns4:test-16916485674093-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-16916485674093,,1691648572496.47fa27f3f64d1137e3edbb66cac8a6d3. 2023-08-10T06:22:53,334 DEBUG [RegionOpenAndInit-ns4:test-16916485674093-pool-0 {}] regionserver.HRegion(1622): Region close journal for 47fa27f3f64d1137e3edbb66cac8a6d3: 2023-08-10T06:22:53,336 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-16916485674093 execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:22:53,336 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-16916485674093,,1691648572496.47fa27f3f64d1137e3edbb66cac8a6d3.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1691648573336"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648573336"}]},"ts":"1691648573336"} 2023-08-10T06:22:53,339 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:22:53,340 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-16916485674093 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:22:53,340 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16916485674093","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648573340"}]},"ts":"1691648573340"} 2023-08-10T06:22:53,341 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16916485674093, state=ENABLING in hbase:meta 2023-08-10T06:22:53,347 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16916485674093, region=47fa27f3f64d1137e3edbb66cac8a6d3, ASSIGN}] 2023-08-10T06:22:53,349 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16916485674093, region=47fa27f3f64d1137e3edbb66cac8a6d3, ASSIGN 2023-08-10T06:22:53,350 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-16916485674093, region=47fa27f3f64d1137e3edbb66cac8a6d3, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37733,1691648558732; forceNewPlan=false, retain=false 2023-08-10T06:22:53,501 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=47fa27f3f64d1137e3edbb66cac8a6d3, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:22:53,505 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure 47fa27f3f64d1137e3edbb66cac8a6d3, server=jenkins-hbase3.apache.org,37733,1691648558732}] 2023-08-10T06:22:53,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-08-10T06:22:53,662 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-16916485674093,,1691648572496.47fa27f3f64d1137e3edbb66cac8a6d3. 2023-08-10T06:22:53,662 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => 47fa27f3f64d1137e3edbb66cac8a6d3, NAME => 'ns4:test-16916485674093,,1691648572496.47fa27f3f64d1137e3edbb66cac8a6d3.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:22:53,663 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:22:53,663 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16916485674093 47fa27f3f64d1137e3edbb66cac8a6d3 2023-08-10T06:22:53,663 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-16916485674093,,1691648572496.47fa27f3f64d1137e3edbb66cac8a6d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:22:53,663 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for 47fa27f3f64d1137e3edbb66cac8a6d3 2023-08-10T06:22:53,663 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for 47fa27f3f64d1137e3edbb66cac8a6d3 2023-08-10T06:22:53,665 INFO [StoreOpener-47fa27f3f64d1137e3edbb66cac8a6d3-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 47fa27f3f64d1137e3edbb66cac8a6d3 2023-08-10T06:22:53,667 INFO [StoreOpener-47fa27f3f64d1137e3edbb66cac8a6d3-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47fa27f3f64d1137e3edbb66cac8a6d3 columnFamilyName f 2023-08-10T06:22:53,667 DEBUG [StoreOpener-47fa27f3f64d1137e3edbb66cac8a6d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:22:53,668 INFO [StoreOpener-47fa27f3f64d1137e3edbb66cac8a6d3-1 {}] regionserver.HStore(324): Store=47fa27f3f64d1137e3edbb66cac8a6d3/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:22:53,669 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/ns4/test-16916485674093/47fa27f3f64d1137e3edbb66cac8a6d3 2023-08-10T06:22:53,670 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/ns4/test-16916485674093/47fa27f3f64d1137e3edbb66cac8a6d3 2023-08-10T06:22:53,674 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for 47fa27f3f64d1137e3edbb66cac8a6d3 2023-08-10T06:22:53,677 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/ns4/test-16916485674093/47fa27f3f64d1137e3edbb66cac8a6d3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:22:53,678 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened 47fa27f3f64d1137e3edbb66cac8a6d3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9607053760, jitterRate=-0.10527339577674866}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:22:53,679 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for 47fa27f3f64d1137e3edbb66cac8a6d3: 2023-08-10T06:22:53,680 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2552): Post open deploy tasks for ns4:test-16916485674093,,1691648572496.47fa27f3f64d1137e3edbb66cac8a6d3., pid=24, masterSystemTime=1691648573657 2023-08-10T06:22:53,683 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2579): Finished post open deploy task for ns4:test-16916485674093,,1691648572496.47fa27f3f64d1137e3edbb66cac8a6d3. 2023-08-10T06:22:53,683 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-16916485674093,,1691648572496.47fa27f3f64d1137e3edbb66cac8a6d3. 2023-08-10T06:22:53,683 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=47fa27f3f64d1137e3edbb66cac8a6d3, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:22:53,688 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=24, resume processing ppid=23 2023-08-10T06:22:53,688 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure 47fa27f3f64d1137e3edbb66cac8a6d3, server=jenkins-hbase3.apache.org,37733,1691648558732 in 182 msec 2023-08-10T06:22:53,691 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=23, resume processing ppid=22 2023-08-10T06:22:53,691 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-16916485674093, region=47fa27f3f64d1137e3edbb66cac8a6d3, ASSIGN in 341 msec 2023-08-10T06:22:53,692 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-16916485674093 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:22:53,692 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16916485674093","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648573692"}]},"ts":"1691648573692"} 2023-08-10T06:22:53,694 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16916485674093, state=ENABLED in hbase:meta 2023-08-10T06:22:53,697 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16916485674093 execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:22:53,699 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-16916485674093 in 1.2010 sec 2023-08-10T06:22:54,227 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:22:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-08-10T06:22:54,610 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns4:test-16916485674093, procId: 22 completed 2023-08-10T06:22:54,610 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-16916485674093 get assigned. Timeout = 60000ms 2023-08-10T06:22:54,611 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-08-10T06:22:54,615 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-16916485674093 assigned to meta. Checking AM states. 2023-08-10T06:22:54,615 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-08-10T06:22:54,615 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-16916485674093 assigned. 2023-08-10T06:22:54,616 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x172202a0 to 127.0.0.1:61601 2023-08-10T06:22:54,616 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:22:54,672 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] hbase.ResourceChecker(147): before: backup.TestBackupDeleteWithFailures#testBackupDeleteWithFailures Thread=583, OpenFileDescriptor=660, MaxFileDescriptor=60000, SystemLoadAverage=209, ProcessCount=173, AvailableMemoryMB=2333 2023-08-10T06:22:54,673 WARN [Listener at jenkins-hbase3.apache.org/36469 {}] hbase.ResourceChecker(130): Thread=583 is superior to 500 2023-08-10T06:22:54,676 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] backup.TestBackupDeleteWithFailures(143): test repair backup delete on a single table with data and failures PRE_DELETE_SNAPSHOT_FAILURE 2023-08-10T06:22:54,676 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x48ef1d43 to 127.0.0.1:61601 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:22:54,692 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44d2f8b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:22:54,704 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:22:54,707 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:57980, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:22:54,723 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:22:54,725 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:40076, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:22:54,738 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'backup'} 2023-08-10T06:22:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure2.ProcedureExecutor(1033): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-08-10T06:22:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=25 2023-08-10T06:22:54,755 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:22:54,760 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=25, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 20 msec 2023-08-10T06:22:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=25 2023-08-10T06:22:54,867 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:22:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure2.ProcedureExecutor(1033): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-08-10T06:22:54,877 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:22:54,877 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:22:54,877 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2023-08-10T06:22:54,879 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:22:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-08-10T06:22:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-08-10T06:22:55,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-08-10T06:22:55,310 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 130db0832f74edda61b4f19f117dbb07, NAME => 'backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322 2023-08-10T06:22:55,322 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:22:55,322 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing 130db0832f74edda61b4f19f117dbb07, disabling compactions & flushes 2023-08-10T06:22:55,322 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:22:55,322 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:22:55,322 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. after waiting 0 ms 2023-08-10T06:22:55,322 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:22:55,322 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:22:55,322 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for 130db0832f74edda61b4f19f117dbb07: 2023-08-10T06:22:55,324 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:22:55,325 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1691648575324"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648575324"}]},"ts":"1691648575324"} 2023-08-10T06:22:55,326 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:22:55,327 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:22:55,328 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648575328"}]},"ts":"1691648575328"} 2023-08-10T06:22:55,329 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-08-10T06:22:55,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=130db0832f74edda61b4f19f117dbb07, ASSIGN}] 2023-08-10T06:22:55,335 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=130db0832f74edda61b4f19f117dbb07, ASSIGN 2023-08-10T06:22:55,336 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=130db0832f74edda61b4f19f117dbb07, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37733,1691648558732; forceNewPlan=false, retain=false 2023-08-10T06:22:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-08-10T06:22:55,488 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=130db0832f74edda61b4f19f117dbb07, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:22:55,499 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; OpenRegionProcedure 130db0832f74edda61b4f19f117dbb07, server=jenkins-hbase3.apache.org,37733,1691648558732}] 2023-08-10T06:22:55,657 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(130): Open backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:22:55,658 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7254): Opening region: {ENCODED => 130db0832f74edda61b4f19f117dbb07, NAME => 'backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:22:55,659 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:22:55,659 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:22:55,659 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(888): Instantiated backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:22:55,659 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7296): checking encryption for 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:22:55,659 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7299): checking classloading for 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:22:55,661 INFO [StoreOpener-130db0832f74edda61b4f19f117dbb07-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:22:55,664 INFO [StoreOpener-130db0832f74edda61b4f19f117dbb07-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 130db0832f74edda61b4f19f117dbb07 columnFamilyName meta 2023-08-10T06:22:55,665 DEBUG [StoreOpener-130db0832f74edda61b4f19f117dbb07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:22:55,665 INFO [StoreOpener-130db0832f74edda61b4f19f117dbb07-1 {}] regionserver.HStore(324): Store=130db0832f74edda61b4f19f117dbb07/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:22:55,666 INFO [StoreOpener-130db0832f74edda61b4f19f117dbb07-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:22:55,668 INFO [StoreOpener-130db0832f74edda61b4f19f117dbb07-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 130db0832f74edda61b4f19f117dbb07 columnFamilyName session 2023-08-10T06:22:55,668 DEBUG [StoreOpener-130db0832f74edda61b4f19f117dbb07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:22:55,668 INFO [StoreOpener-130db0832f74edda61b4f19f117dbb07-1 {}] regionserver.HStore(324): Store=130db0832f74edda61b4f19f117dbb07/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:22:55,669 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07 2023-08-10T06:22:55,670 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07 2023-08-10T06:22:55,672 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-08-10T06:22:55,673 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1076): writing seq id for 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:22:55,676 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:22:55,677 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): Opened 130db0832f74edda61b4f19f117dbb07; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11102945120, jitterRate=0.034042343497276306}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-08-10T06:22:55,677 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(992): Region open journal for 130db0832f74edda61b4f19f117dbb07: 2023-08-10T06:22:55,679 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07., pid=28, masterSystemTime=1691648575652 2023-08-10T06:22:55,682 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:22:55,682 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(158): Opened backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:22:55,683 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=130db0832f74edda61b4f19f117dbb07, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:22:55,688 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=28, resume processing ppid=27 2023-08-10T06:22:55,688 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=28, ppid=27, state=SUCCESS; OpenRegionProcedure 130db0832f74edda61b4f19f117dbb07, server=jenkins-hbase3.apache.org,37733,1691648558732 in 186 msec 2023-08-10T06:22:55,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=27, resume processing ppid=26 2023-08-10T06:22:55,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=130db0832f74edda61b4f19f117dbb07, ASSIGN in 355 msec 2023-08-10T06:22:55,693 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:22:55,693 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648575693"}]},"ts":"1691648575693"} 2023-08-10T06:22:55,695 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-08-10T06:22:55,700 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:22:55,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=26, state=SUCCESS; CreateTableProcedure table=backup:system in 833 msec 2023-08-10T06:22:55,864 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-08-10T06:22:55,864 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-08-10T06:22:55,865 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16916485674093 2023-08-10T06:22:55,865 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-16916485674093 Metrics about Tables on a single HBase RegionServer 2023-08-10T06:22:55,866 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1691648567409 2023-08-10T06:22:55,866 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1691648567409 Metrics about Tables on a single HBase RegionServer 2023-08-10T06:22:55,867 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16916485674092 2023-08-10T06:22:55,867 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-16916485674092 Metrics about Tables on a single HBase RegionServer 2023-08-10T06:22:55,868 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16916485674091 2023-08-10T06:22:55,868 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-16916485674091 Metrics about Tables on a single HBase RegionServer 2023-08-10T06:22:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-08-10T06:22:55,991 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: backup:system, procId: 26 completed 2023-08-10T06:22:56,005 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:22:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure2.ProcedureExecutor(1033): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-08-10T06:22:56,009 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:22:56,009 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:22:56,010 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2023-08-10T06:22:56,011 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:22:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-08-10T06:22:56,026 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => aebeeaff0240a9e4a35acc504f682275, NAME => 'backup:system_bulk,,1691648576005.aebeeaff0240a9e4a35acc504f682275.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322 2023-08-10T06:22:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-08-10T06:22:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-08-10T06:22:56,440 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1691648576005.aebeeaff0240a9e4a35acc504f682275.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:22:56,440 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing aebeeaff0240a9e4a35acc504f682275, disabling compactions & flushes 2023-08-10T06:22:56,440 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1691648576005.aebeeaff0240a9e4a35acc504f682275. 2023-08-10T06:22:56,440 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1691648576005.aebeeaff0240a9e4a35acc504f682275. 2023-08-10T06:22:56,441 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1691648576005.aebeeaff0240a9e4a35acc504f682275. after waiting 0 ms 2023-08-10T06:22:56,441 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1691648576005.aebeeaff0240a9e4a35acc504f682275. 2023-08-10T06:22:56,441 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1691648576005.aebeeaff0240a9e4a35acc504f682275. 2023-08-10T06:22:56,441 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for aebeeaff0240a9e4a35acc504f682275: 2023-08-10T06:22:56,442 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:22:56,443 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1691648576005.aebeeaff0240a9e4a35acc504f682275.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1691648576442"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648576442"}]},"ts":"1691648576442"} 2023-08-10T06:22:56,445 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:22:56,446 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:22:56,446 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648576446"}]},"ts":"1691648576446"} 2023-08-10T06:22:56,448 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-08-10T06:22:56,453 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=aebeeaff0240a9e4a35acc504f682275, ASSIGN}] 2023-08-10T06:22:56,455 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=aebeeaff0240a9e4a35acc504f682275, ASSIGN 2023-08-10T06:22:56,456 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=aebeeaff0240a9e4a35acc504f682275, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37733,1691648558732; forceNewPlan=false, retain=false 2023-08-10T06:22:56,608 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=aebeeaff0240a9e4a35acc504f682275, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:22:56,611 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE; OpenRegionProcedure aebeeaff0240a9e4a35acc504f682275, server=jenkins-hbase3.apache.org,37733,1691648558732}] 2023-08-10T06:22:56,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-08-10T06:22:56,769 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1691648576005.aebeeaff0240a9e4a35acc504f682275. 2023-08-10T06:22:56,770 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7254): Opening region: {ENCODED => aebeeaff0240a9e4a35acc504f682275, NAME => 'backup:system_bulk,,1691648576005.aebeeaff0240a9e4a35acc504f682275.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:22:56,770 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:22:56,771 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk aebeeaff0240a9e4a35acc504f682275 2023-08-10T06:22:56,771 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1691648576005.aebeeaff0240a9e4a35acc504f682275.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:22:56,771 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7296): checking encryption for aebeeaff0240a9e4a35acc504f682275 2023-08-10T06:22:56,771 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7299): checking classloading for aebeeaff0240a9e4a35acc504f682275 2023-08-10T06:22:56,773 INFO [StoreOpener-aebeeaff0240a9e4a35acc504f682275-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region aebeeaff0240a9e4a35acc504f682275 2023-08-10T06:22:56,775 INFO [StoreOpener-aebeeaff0240a9e4a35acc504f682275-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aebeeaff0240a9e4a35acc504f682275 columnFamilyName meta 2023-08-10T06:22:56,775 DEBUG [StoreOpener-aebeeaff0240a9e4a35acc504f682275-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:22:56,776 INFO [StoreOpener-aebeeaff0240a9e4a35acc504f682275-1 {}] regionserver.HStore(324): Store=aebeeaff0240a9e4a35acc504f682275/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:22:56,776 INFO [StoreOpener-aebeeaff0240a9e4a35acc504f682275-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region aebeeaff0240a9e4a35acc504f682275 2023-08-10T06:22:56,778 INFO [StoreOpener-aebeeaff0240a9e4a35acc504f682275-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aebeeaff0240a9e4a35acc504f682275 columnFamilyName session 2023-08-10T06:22:56,778 DEBUG [StoreOpener-aebeeaff0240a9e4a35acc504f682275-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:22:56,779 INFO [StoreOpener-aebeeaff0240a9e4a35acc504f682275-1 {}] regionserver.HStore(324): Store=aebeeaff0240a9e4a35acc504f682275/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:22:56,780 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system_bulk/aebeeaff0240a9e4a35acc504f682275 2023-08-10T06:22:56,781 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system_bulk/aebeeaff0240a9e4a35acc504f682275 2023-08-10T06:22:56,783 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-08-10T06:22:56,785 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1076): writing seq id for aebeeaff0240a9e4a35acc504f682275 2023-08-10T06:22:56,789 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system_bulk/aebeeaff0240a9e4a35acc504f682275/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:22:56,790 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): Opened aebeeaff0240a9e4a35acc504f682275; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10877601920, jitterRate=0.013055622577667236}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-08-10T06:22:56,790 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(992): Region open journal for aebeeaff0240a9e4a35acc504f682275: 2023-08-10T06:22:56,791 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system_bulk,,1691648576005.aebeeaff0240a9e4a35acc504f682275., pid=31, masterSystemTime=1691648576764 2023-08-10T06:22:56,794 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system_bulk,,1691648576005.aebeeaff0240a9e4a35acc504f682275. 2023-08-10T06:22:56,794 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1691648576005.aebeeaff0240a9e4a35acc504f682275. 2023-08-10T06:22:56,795 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=aebeeaff0240a9e4a35acc504f682275, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:22:56,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=31, resume processing ppid=30 2023-08-10T06:22:56,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=31, ppid=30, state=SUCCESS; OpenRegionProcedure aebeeaff0240a9e4a35acc504f682275, server=jenkins-hbase3.apache.org,37733,1691648558732 in 186 msec 2023-08-10T06:22:56,812 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=30, resume processing ppid=29 2023-08-10T06:22:56,812 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=30, ppid=29, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=aebeeaff0240a9e4a35acc504f682275, ASSIGN in 356 msec 2023-08-10T06:22:56,813 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:22:56,813 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648576813"}]},"ts":"1691648576813"} 2023-08-10T06:22:56,816 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-08-10T06:22:56,819 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:22:56,823 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=29, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 814 msec 2023-08-10T06:22:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-08-10T06:22:57,118 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: backup:system_bulk, procId: 29 completed 2023-08-10T06:22:57,131 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:22:57,179 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:22:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648577179 (current time:1691648577179). 2023-08-10T06:22:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:22:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:22:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0df1153f to 127.0.0.1:61601 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:22:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b7e1270, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:22:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:22:57,193 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:57992, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:22:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0df1153f to 127.0.0.1:61601 2023-08-10T06:22:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:22:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:22:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure2.ProcedureExecutor(1033): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:22:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-08-10T06:22:57,211 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:22:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-08-10T06:22:57,217 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:22:57,219 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:22:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-08-10T06:22:57,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-08-10T06:22:57,630 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:22:57,633 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 130db0832f74edda61b4f19f117dbb07}] 2023-08-10T06:22:57,641 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:22:57,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37733 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2023-08-10T06:22:57,803 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:22:57,806 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2819): Flushing 130db0832f74edda61b4f19f117dbb07 2/2 column families, dataSize=45 B heapSize=632 B 2023-08-10T06:22:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-08-10T06:22:57,881 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/session/840b366be1f149a6a0f25329a1515460 is 49, key is activesession:/session:c/1691648577162/Put/seqid=0 2023-08-10T06:22:58,301 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/session/840b366be1f149a6a0f25329a1515460 2023-08-10T06:22:58,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-08-10T06:22:58,367 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 840b366be1f149a6a0f25329a1515460 2023-08-10T06:22:58,370 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/session/840b366be1f149a6a0f25329a1515460 as hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/840b366be1f149a6a0f25329a1515460 2023-08-10T06:22:58,378 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/840b366be1f149a6a0f25329a1515460, entries=1, sequenceid=5, filesize=4.9 K 2023-08-10T06:22:58,382 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for 130db0832f74edda61b4f19f117dbb07 in 577ms, sequenceid=5, compaction requested=false 2023-08-10T06:22:58,382 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-08-10T06:22:58,384 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2520): Flush status journal for 130db0832f74edda61b4f19f117dbb07: 2023-08-10T06:22:58,384 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. for snapshot_backup_system completed. 2023-08-10T06:22:58,385 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:22:58,392 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:22:58,397 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-08-10T06:22:58,397 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/840b366be1f149a6a0f25329a1515460] hfiles 2023-08-10T06:22:58,397 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/840b366be1f149a6a0f25329a1515460 for snapshot=snapshot_backup_system 2023-08-10T06:22:58,816 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:22:58,819 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=33 2023-08-10T06:22:58,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.HMaster(3994): Remote procedure done, pid=33 2023-08-10T06:22:58,823 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:22:58,825 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:22:58,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=33, resume processing ppid=32 2023-08-10T06:22:58,828 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:22:58,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=33, ppid=32, state=SUCCESS; SnapshotRegionProcedure 130db0832f74edda61b4f19f117dbb07 in 1.1930 sec 2023-08-10T06:22:58,830 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:22:58,834 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:22:58,834 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:22:58,837 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:22:59,167 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:22:59,223 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-08-10T06:22:59,224 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-16916485674093' 2023-08-10T06:22:59,262 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:22:59,276 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:22:59,276 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:22:59,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-08-10T06:22:59,712 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:22:59,712 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-08-10T06:22:59,715 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=32, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.5100 sec 2023-08-10T06:23:01,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-08-10T06:23:01,321 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 32 completed 2023-08-10T06:23:01,322 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] impl.TableBackupClient(115): Backup backup_1691648574696 started at 1691648581321. 2023-08-10T06:23:01,344 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.TableBackupClient(119): Backup session backup_1691648574696 has been started. 2023-08-10T06:23:01,350 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-08-10T06:23:01,362 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-08-10T06:23:01,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-08-10T06:23:01,369 INFO [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-08-10T06:23:01,369 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-08-10T06:23:01,370 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-08-10T06:23:01,370 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-08-10T06:23:01,370 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-08-10T06:23:01,370 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:23:01,372 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37733-0x10060e5d75b0001, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-08-10T06:23:01,372 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:01,372 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:23:01,372 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:23:01,372 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:01,372 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-08-10T06:23:01,372 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-08-10T06:23:01,373 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37733-0x10060e5d75b0001, quorum=127.0.0.1:61601, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-08-10T06:23:01,373 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-08-10T06:23:01,373 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:23:01,373 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-08-10T06:23:01,375 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-08-10T06:23:01,375 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-08-10T06:23:01,376 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-08-10T06:23:01,376 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-08-10T06:23:01,377 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-08-10T06:23:01,377 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-08-10T06:23:01,377 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,37733,1691648558732' joining acquired barrier for procedure (rolllog) in zk 2023-08-10T06:23:01,379 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:01,379 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-08-10T06:23:01,379 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:01,379 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:23:01,379 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:23:01,379 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37733-0x10060e5d75b0001, quorum=127.0.0.1:61601, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-08-10T06:23:01,379 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-08-10T06:23:01,379 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:23:01,380 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:23:01,380 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:23:01,380 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:01,381 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:23:01,381 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,37733,1691648558732' joining acquired barrier for procedure 'rolllog' on coordinator 2023-08-10T06:23:01,381 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@c46e325[Count = 0] remaining members to acquire global barrier 2023-08-10T06:23:01,381 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-08-10T06:23:01,381 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-08-10T06:23:01,383 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37733-0x10060e5d75b0001, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:23:01,383 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-08-10T06:23:01,383 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-08-10T06:23:01,383 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-08-10T06:23:01,383 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:01,383 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-08-10T06:23:01,384 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-08-10T06:23:01,384 DEBUG [rs(jenkins-hbase3.apache.org,37733,1691648558732)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:01,384 INFO [rs(jenkins-hbase3.apache.org,37733,1691648558732)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1691648560631 highest: 1691648560631 on jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:01,384 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37733%2C1691648558732:(num 1691648560631) roll requested 2023-08-10T06:23:01,422 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.1691648581385, exclude list is [], retry=0 2023-08-10T06:23:01,426 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45175,DS-cb72d2f9-6999-4dc2-81ce-221dee4d930e,DISK] 2023-08-10T06:23:01,430 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.1691648560631 with entries=19, filesize=3.99 KB; new WAL /user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.1691648581385 2023-08-10T06:23:01,431 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:45175,DS-cb72d2f9-6999-4dc2-81ce-221dee4d930e,DISK]] 2023-08-10T06:23:01,432 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.1691648560631 is not closed yet, will try archiving it next time 2023-08-10T06:23:01,432 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37733%2C1691648558732.meta:.meta(num 1691648560998) roll requested 2023-08-10T06:23:01,445 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.meta.1691648581432.meta, exclude list is [], retry=0 2023-08-10T06:23:01,449 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45175,DS-cb72d2f9-6999-4dc2-81ce-221dee4d930e,DISK] 2023-08-10T06:23:01,451 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.meta.1691648560998.meta with entries=36, filesize=11.13 KB; new WAL /user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.meta.1691648581432.meta 2023-08-10T06:23:01,451 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:45175,DS-cb72d2f9-6999-4dc2-81ce-221dee4d930e,DISK]] 2023-08-10T06:23:01,451 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.meta.1691648560998.meta is not closed yet, will try archiving it next time 2023-08-10T06:23:01,455 DEBUG [rs(jenkins-hbase3.apache.org,37733,1691648558732)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 71 2023-08-10T06:23:01,455 INFO [rs(jenkins-hbase3.apache.org,37733,1691648558732)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1691648581385 on jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:01,456 DEBUG [rs(jenkins-hbase3.apache.org,37733,1691648558732)-backup-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:23:01,457 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:44570, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-08-10T06:23:01,475 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-08-10T06:23:01,475 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,37733,1691648558732' in zk 2023-08-10T06:23:01,479 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:01,479 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-08-10T06:23:01,479 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:23:01,479 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:01,479 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-08-10T06:23:01,480 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:23:01,480 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:23:01,480 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:23:01,480 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:23:01,481 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:23:01,481 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:01,481 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:23:01,482 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:23:01,482 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:01,483 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,37733,1691648558732': 2023-08-10T06:23:01,483 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,37733,1691648558732' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-08-10T06:23:01,483 INFO [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-08-10T06:23:01,483 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-08-10T06:23:01,483 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-08-10T06:23:01,483 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-08-10T06:23:01,483 INFO [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-08-10T06:23:01,485 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37733-0x10060e5d75b0001, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:23:01,486 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37733-0x10060e5d75b0001, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-08-10T06:23:01,486 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-08-10T06:23:01,486 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:23:01,486 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:23:01,486 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-08-10T06:23:01,486 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:23:01,486 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:23:01,486 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:23:01,486 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:23:01,486 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:23:01,486 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:01,487 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:23:01,487 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:23:01,487 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:23:01,488 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:23:01,488 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:01,488 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:01,488 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:23:01,489 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:23:01,489 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:01,496 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:01,496 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-08-10T06:23:01,496 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37733-0x10060e5d75b0001, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-08-10T06:23:01,496 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-08-10T06:23:01,496 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37733-0x10060e5d75b0001, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-08-10T06:23:01,496 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:23:01,497 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:01,497 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:23:01,497 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:23:01,497 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:23:01,497 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-08-10T06:23:01,497 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:23:01,497 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-08-10T06:23:01,497 DEBUG [Listener at localhost/39747-EventThread {}] zookeeper.ZKWatcher(604): master:39523-0x10060e5d75b0000, quorum=127.0.0.1:61601, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:23:01,498 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:23:01,498 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:23:01,499 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] client.HBaseAdmin(2751): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 100000 ms per retry) 2023-08-10T06:23:01,499 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] client.HBaseAdmin(2759): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-08-10T06:23:01,600 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] client.HBaseAdmin(2765): Getting current status of procedure from master... 2023-08-10T06:23:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-08-10T06:23:01,609 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1691648581608_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } 2023-08-10T06:23:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648581609 (current time:1691648581609). 2023-08-10T06:23:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:23:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1691648581608_default_test-1691648567409 VERSION not specified, setting to 2 2023-08-10T06:23:01,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x24251159 to 127.0.0.1:61601 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:23:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76bfdbfb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:23:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:23:01,619 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:47278, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:23:01,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x24251159 to 127.0.0.1:61601 2023-08-10T06:23:01,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:23:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:23:01,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure2.ProcedureExecutor(1033): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691648581608_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } 2023-08-10T06:23:01,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1691648581608_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-08-10T06:23:01,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-08-10T06:23:01,624 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691648581608_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:23:01,625 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691648581608_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:23:01,628 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691648581608_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:23:01,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-08-10T06:23:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-08-10T06:23:02,040 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691648581608_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:23:02,040 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 89ac3e8e753d2c1222fc1445a7518dbe}] 2023-08-10T06:23:02,042 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 89ac3e8e753d2c1222fc1445a7518dbe 2023-08-10T06:23:02,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37733 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2023-08-10T06:23:02,195 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe. 2023-08-10T06:23:02,196 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2819): Flushing 89ac3e8e753d2c1222fc1445a7518dbe 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-08-10T06:23:02,221 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/default/test-1691648567409/89ac3e8e753d2c1222fc1445a7518dbe/.tmp/f/6ec84d0138b7498790d3e4f4273b9498 is 37, key is row10/f:q1/1691648569068/Put/seqid=0 2023-08-10T06:23:02,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-08-10T06:23:02,239 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/default/test-1691648567409/89ac3e8e753d2c1222fc1445a7518dbe/.tmp/f/6ec84d0138b7498790d3e4f4273b9498 2023-08-10T06:23:02,249 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6ec84d0138b7498790d3e4f4273b9498 2023-08-10T06:23:02,251 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/default/test-1691648567409/89ac3e8e753d2c1222fc1445a7518dbe/.tmp/f/6ec84d0138b7498790d3e4f4273b9498 as hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/default/test-1691648567409/89ac3e8e753d2c1222fc1445a7518dbe/f/6ec84d0138b7498790d3e4f4273b9498 2023-08-10T06:23:02,259 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/default/test-1691648567409/89ac3e8e753d2c1222fc1445a7518dbe/f/6ec84d0138b7498790d3e4f4273b9498, entries=99, sequenceid=103, filesize=8.2 K 2023-08-10T06:23:02,261 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 89ac3e8e753d2c1222fc1445a7518dbe in 64ms, sequenceid=103, compaction requested=false 2023-08-10T06:23:02,261 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2520): Flush status journal for 89ac3e8e753d2c1222fc1445a7518dbe: 2023-08-10T06:23:02,261 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe. for snapshot_1691648581608_default_test-1691648567409 completed. 2023-08-10T06:23:02,262 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(240): Storing 'test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe.' region-info for snapshot=snapshot_1691648581608_default_test-1691648567409 2023-08-10T06:23:02,262 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:23:02,262 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/default/test-1691648567409/89ac3e8e753d2c1222fc1445a7518dbe/f/6ec84d0138b7498790d3e4f4273b9498] hfiles 2023-08-10T06:23:02,262 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/default/test-1691648567409/89ac3e8e753d2c1222fc1445a7518dbe/f/6ec84d0138b7498790d3e4f4273b9498 for snapshot=snapshot_1691648581608_default_test-1691648567409 2023-08-10T06:23:02,672 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe. 2023-08-10T06:23:02,672 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=35 2023-08-10T06:23:02,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.HMaster(3994): Remote procedure done, pid=35 2023-08-10T06:23:02,673 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1691648581608_default_test-1691648567409 on region 89ac3e8e753d2c1222fc1445a7518dbe 2023-08-10T06:23:02,674 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 89ac3e8e753d2c1222fc1445a7518dbe 2023-08-10T06:23:02,678 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=35, resume processing ppid=34 2023-08-10T06:23:02,678 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691648581608_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:23:02,678 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=35, ppid=34, state=SUCCESS; SnapshotRegionProcedure 89ac3e8e753d2c1222fc1445a7518dbe in 636 msec 2023-08-10T06:23:02,679 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691648581608_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:23:02,679 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691648581608_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:23:02,679 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1691648581608_default_test-1691648567409 2023-08-10T06:23:02,680 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/.tmp/snapshot_1691648581608_default_test-1691648567409 2023-08-10T06:23:02,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-08-10T06:23:03,097 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691648581608_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:23:03,105 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691648581608_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:23:03,106 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/.tmp/snapshot_1691648581608_default_test-1691648567409 to hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/snapshot_1691648581608_default_test-1691648567409 2023-08-10T06:23:03,532 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691648581608_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:23:03,532 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1691648581608_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-08-10T06:23:03,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=34, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1691648581608_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } in 1.9110 sec 2023-08-10T06:23:03,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-08-10T06:23:03,733 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: default:test-1691648567409, procId: 34 completed 2023-08-10T06:23:03,733 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1691648574696 2023-08-10T06:23:03,733 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-08-10T06:23:03,735 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-08-10T06:23:03,737 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1691648574696_test-1691648567409 2023-08-10T06:23:03,737 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1691648581608_default_test-1691648567409 to hdfs://localhost:46823/backupUT/backup_1691648574696/default/test-1691648567409/ 2023-08-10T06:23:03,747 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-08-10T06:23:03,779 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:46823, inputRoot=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322 2023-08-10T06:23:03,779 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,39523,1691648555675_-2003647586_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:46823/backupUT/backup_1691648574696/default/test-1691648567409, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:46823/backupUT/backup_1691648574696/default/test-1691648567409/.hbase-snapshot/.tmp/snapshot_1691648581608_default_test-1691648567409 2023-08-10T06:23:03,779 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:46823, inputRoot=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322, snapshotDir=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/snapshot_1691648581608_default_test-1691648567409. 2023-08-10T06:23:03,791 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/snapshot_1691648581608_default_test-1691648567409 to hdfs://localhost:46823/backupUT/backup_1691648574696/default/test-1691648567409/.hbase-snapshot/.tmp/snapshot_1691648581608_default_test-1691648567409 2023-08-10T06:23:04,232 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:04,233 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:04,234 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:04,234 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:05,236 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/hadoop-2898841211009508452.jar 2023-08-10T06:23:05,237 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:05,237 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:05,237 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:05,238 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:05,238 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:05,239 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:05,239 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:05,240 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:05,240 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:05,241 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:23:05,241 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:23:05,242 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:23:05,242 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:23:05,243 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:23:05,243 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:23:05,244 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:23:05,244 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:23:05,245 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:23:05,245 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:23:05,246 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:23:05,247 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:23:05,251 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:23:05,252 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:23:05,252 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:23:05,252 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:23:05,253 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:23:05,253 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:23:05,254 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:23:05,312 WARN [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-08-10T06:23:05,335 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1691648581608_default_test-1691648567409' hfile list 2023-08-10T06:23:05,345 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-08-10T06:23:05,563 WARN [Listener at jenkins-hbase3.apache.org/36469 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/mapreduce.cluster.local.dir/1691648585516/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:23:05,564 WARN [Listener at jenkins-hbase3.apache.org/36469 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/mapreduce.cluster.local.dir/1691648585516/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:23:05,712 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-08-10T06:23:05,748 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/default/test-1691648567409/89ac3e8e753d2c1222fc1445a7518dbe/f/6ec84d0138b7498790d3e4f4273b9498 output=hdfs://localhost:46823/backupUT/backup_1691648574696/default/test-1691648567409/archive/data/default/test-1691648567409/89ac3e8e753d2c1222fc1445a7518dbe/f/6ec84d0138b7498790d3e4f4273b9498 2023-08-10T06:23:05,748 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-08-10T06:23:05,864 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-08-10T06:23:05,864 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-08-10T06:23:06,620 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-08-10T06:23:06,621 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-08-10T06:23:06,629 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1691648581608_default_test-1691648567409 2023-08-10T06:23:06,629 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1691648581608_default_test-1691648567409 finished. 2023-08-10T06:23:06,629 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1014): test-1691648567409 2023-08-10T06:23:06,700 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691648574696 2023-08-10T06:23:06,700 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-08-10T06:23:07,115 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:46823/backupUT/backup_1691648574696/.backup.manifest 2023-08-10T06:23:07,115 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.TableBackupClient(358): Backup backup_1691648574696 finished: type=FULL,tablelist=test-1691648567409,targetRootDir=hdfs://localhost:46823/backupUT,startts=1691648581321,completets=1691648586697,bytescopied=0 2023-08-10T06:23:07,115 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-08-10T06:23:07,115 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1691648581608_default_test-1691648567409 2023-08-10T06:23:07,126 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1691648581608_default_test-1691648567409" 2023-08-10T06:23:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1691648581608_default_test-1691648567409 2023-08-10T06:23:07,132 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1691648581608_default_test-1691648567409 for backup backup_1691648574696 succeeded. 2023-08-10T06:23:07,133 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:23:07,145 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:23:07,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:23:07,148 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:23:07,155 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:23:07,172 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] impl.TableBackupClient(376): Backup backup_1691648574696 completed. 2023-08-10T06:23:07,172 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] client.ConnectionImplementation(2114): Closing master protocol: MasterService 2023-08-10T06:23:07,173 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x48ef1d43 to 127.0.0.1:61601 2023-08-10T06:23:07,173 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:23:07,192 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] backup.TestBackupDeleteWithFailures(147): backup complete 2023-08-10T06:23:07,217 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:23:07,226 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:23:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648587226 (current time:1691648587226). 2023-08-10T06:23:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:23:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:23:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1582b016 to 127.0.0.1:61601 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:23:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cfb3df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:23:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:23:07,235 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:47286, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:23:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1582b016 to 127.0.0.1:61601 2023-08-10T06:23:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:23:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:23:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure2.ProcedureExecutor(1033): Stored pid=36, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:23:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-08-10T06:23:07,240 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:23:07,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-08-10T06:23:07,241 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:23:07,243 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:23:07,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-08-10T06:23:07,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-08-10T06:23:07,652 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:23:07,652 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 130db0832f74edda61b4f19f117dbb07}] 2023-08-10T06:23:07,654 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:07,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37733 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=37 2023-08-10T06:23:07,806 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:23:07,807 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2819): Flushing 130db0832f74edda61b4f19f117dbb07 2/2 column families, dataSize=1.14 KB heapSize=2.37 KB 2023-08-10T06:23:07,822 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/meta/6d6621fec99f4115886e4e523164d414 is 169, key is trslm:hdfs://localhost:46823/backupUT\x00test-1691648567409/meta:log-roll-map/1691648586687/Put/seqid=0 2023-08-10T06:23:07,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-08-10T06:23:08,227 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=586 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/meta/6d6621fec99f4115886e4e523164d414 2023-08-10T06:23:08,234 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6d6621fec99f4115886e4e523164d414 2023-08-10T06:23:08,238 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/session/6f9e71fc2fea41d49c5fddcb29e166d7 is 310, key is session:backup_1691648574696/session:context/1691648587154/Put/seqid=0 2023-08-10T06:23:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-08-10T06:23:08,644 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=579 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/session/6f9e71fc2fea41d49c5fddcb29e166d7 2023-08-10T06:23:08,651 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6f9e71fc2fea41d49c5fddcb29e166d7 2023-08-10T06:23:08,653 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/meta/6d6621fec99f4115886e4e523164d414 as hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/6d6621fec99f4115886e4e523164d414 2023-08-10T06:23:08,661 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/6d6621fec99f4115886e4e523164d414, entries=5, sequenceid=18, filesize=5.5 K 2023-08-10T06:23:08,662 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/session/6f9e71fc2fea41d49c5fddcb29e166d7 as hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/6f9e71fc2fea41d49c5fddcb29e166d7 2023-08-10T06:23:08,669 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/6f9e71fc2fea41d49c5fddcb29e166d7, entries=2, sequenceid=18, filesize=5.3 K 2023-08-10T06:23:08,671 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1165, heapSize ~2.34 KB/2392, currentSize=0 B/0 for 130db0832f74edda61b4f19f117dbb07 in 864ms, sequenceid=18, compaction requested=false 2023-08-10T06:23:08,671 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2520): Flush status journal for 130db0832f74edda61b4f19f117dbb07: 2023-08-10T06:23:08,671 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. for snapshot_backup_system completed. 2023-08-10T06:23:08,671 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:23:08,671 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:23:08,671 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/6d6621fec99f4115886e4e523164d414] hfiles 2023-08-10T06:23:08,672 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/6d6621fec99f4115886e4e523164d414 for snapshot=snapshot_backup_system 2023-08-10T06:23:08,672 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/840b366be1f149a6a0f25329a1515460, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/6f9e71fc2fea41d49c5fddcb29e166d7] hfiles 2023-08-10T06:23:08,672 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/840b366be1f149a6a0f25329a1515460 for snapshot=snapshot_backup_system 2023-08-10T06:23:08,672 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/6f9e71fc2fea41d49c5fddcb29e166d7 for snapshot=snapshot_backup_system 2023-08-10T06:23:09,078 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:23:09,079 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=37 2023-08-10T06:23:09,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.HMaster(3994): Remote procedure done, pid=37 2023-08-10T06:23:09,079 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:09,081 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:09,084 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=37, resume processing ppid=36 2023-08-10T06:23:09,084 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:23:09,084 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=37, ppid=36, state=SUCCESS; SnapshotRegionProcedure 130db0832f74edda61b4f19f117dbb07 in 1.4300 sec 2023-08-10T06:23:09,085 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:23:09,085 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:23:09,085 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:23:09,086 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:23:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-08-10T06:23:09,498 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:23:09,507 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:23:09,507 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:23:10,061 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-08-10T06:23:10,063 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:44584, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-08-10T06:23:10,328 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:23:10,329 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-08-10T06:23:10,331 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=36, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0920 sec 2023-08-10T06:23:11,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=36 2023-08-10T06:23:11,349 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 36 completed 2023-08-10T06:23:11,356 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupAdminImpl(227): Deleting backup backup_1691648574696 ... 2023-08-10T06:23:11,357 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] util.BackupUtils(435): Delete log files: .backup.manifest 2023-08-10T06:23:11,358 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] util.BackupUtils(435): Delete log files: default 2023-08-10T06:23:11,359 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] util.BackupUtils(443): Trying to cleanup up target dir : backup_1691648574696 2023-08-10T06:23:11,360 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] util.BackupUtils(458): No data has been found in hdfs://localhost:46823/backupUT/backup_1691648574696/default/test-1691648567409. 2023-08-10T06:23:11,363 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] util.BackupUtils(465): hdfs://localhost:46823/backupUT/backup_1691648574696/default is empty, remove it. 2023-08-10T06:23:11,372 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupAdminImpl(269): 0 bulk loaded files out of 0 were deleted 2023-08-10T06:23:11,372 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(433): deleted 0 original bulkload rows 2023-08-10T06:23:11,375 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupAdminImpl(276): Delete backup backup_1691648574696 completed. 2023-08-10T06:23:11,382 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1014): test-1691648567409 2023-08-10T06:23:11,385 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:23:11,390 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:23:11,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.MetricsHBaseServer(144): Unknown exception type java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1155) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:375) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:760) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-08-10T06:23:11,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.CallRunner(138): callId: 147 service: MasterService methodName: DeleteSnapshot size: 57 connection: 172.31.12.81:48964 deadline: 1691648651390, exception=java.io.IOException: preDeleteSnapshot 2023-08-10T06:23:11,496 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:23:11,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.MetricsHBaseServer(144): Unknown exception type java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1155) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:375) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:760) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-08-10T06:23:11,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.CallRunner(138): callId: 149 service: MasterService methodName: DeleteSnapshot size: 57 connection: 172.31.12.81:48964 deadline: 1691648651495, exception=java.io.IOException: preDeleteSnapshot 2023-08-10T06:23:11,505 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-08-10T06:23:11,510 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] client.HBaseAdmin$18(963): Started disable of backup:system 2023-08-10T06:23:11,516 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable backup:system 2023-08-10T06:23:11,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure2.ProcedureExecutor(1033): Stored pid=38, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-08-10T06:23:11,528 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648591528"}]},"ts":"1691648591528"} 2023-08-10T06:23:11,530 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-08-10T06:23:11,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-08-10T06:23:11,532 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-08-10T06:23:11,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=130db0832f74edda61b4f19f117dbb07, UNASSIGN}] 2023-08-10T06:23:11,536 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=130db0832f74edda61b4f19f117dbb07, UNASSIGN 2023-08-10T06:23:11,537 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=39 updating hbase:meta row=130db0832f74edda61b4f19f117dbb07, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:11,539 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE; CloseRegionProcedure 130db0832f74edda61b4f19f117dbb07, server=jenkins-hbase3.apache.org,37733,1691648558732}] 2023-08-10T06:23:11,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-08-10T06:23:11,700 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(119): Close 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:11,700 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1668): Closing 130db0832f74edda61b4f19f117dbb07, disabling compactions & flushes 2023-08-10T06:23:11,700 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1690): Closing region backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:23:11,700 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:23:11,700 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. after waiting 0 ms 2023-08-10T06:23:11,700 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:23:11,700 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(2819): Flushing 130db0832f74edda61b4f19f117dbb07 2/2 column families, dataSize=179 B heapSize=912 B 2023-08-10T06:23:11,705 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/meta/37038161122946e9af290add645e1bb6 is 91, key is incrbackupset:hdfs://localhost:46823/backupUT/meta:test-1691648567409/1691648591382/Put/seqid=0 2023-08-10T06:23:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-08-10T06:23:12,110 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=124 B at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/meta/37038161122946e9af290add645e1bb6 2023-08-10T06:23:12,118 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 37038161122946e9af290add645e1bb6 2023-08-10T06:23:12,118 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 37038161122946e9af290add645e1bb6 2023-08-10T06:23:12,121 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/session/fd5f4ea6e4a040f0bdfced6ae21741da is 59, key is session:backup_1691648574696/session:/1691648591374/DeleteFamily/seqid=0 2023-08-10T06:23:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-08-10T06:23:12,526 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55 B at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/session/fd5f4ea6e4a040f0bdfced6ae21741da 2023-08-10T06:23:12,533 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fd5f4ea6e4a040f0bdfced6ae21741da 2023-08-10T06:23:12,533 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: fd5f4ea6e4a040f0bdfced6ae21741da 2023-08-10T06:23:12,534 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/meta/37038161122946e9af290add645e1bb6 as hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/37038161122946e9af290add645e1bb6 2023-08-10T06:23:12,541 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 37038161122946e9af290add645e1bb6 2023-08-10T06:23:12,541 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/37038161122946e9af290add645e1bb6, entries=2, sequenceid=24, filesize=5.2 K 2023-08-10T06:23:12,543 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/session/fd5f4ea6e4a040f0bdfced6ae21741da as hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/fd5f4ea6e4a040f0bdfced6ae21741da 2023-08-10T06:23:12,549 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fd5f4ea6e4a040f0bdfced6ae21741da 2023-08-10T06:23:12,550 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/fd5f4ea6e4a040f0bdfced6ae21741da, entries=1, sequenceid=24, filesize=5.1 K 2023-08-10T06:23:12,551 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(3022): Finished flush of dataSize ~179 B/179, heapSize ~880 B/880, currentSize=0 B/0 for 130db0832f74edda61b4f19f117dbb07 in 851ms, sequenceid=24, compaction requested=true 2023-08-10T06:23:12,553 DEBUG [StoreFileCloser-130db0832f74edda61b4f19f117dbb07-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6d6621fec99f4115886e4e523164d414 2023-08-10T06:23:12,555 DEBUG [StoreFileCloser-130db0832f74edda61b4f19f117dbb07-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 37038161122946e9af290add645e1bb6 2023-08-10T06:23:12,557 DEBUG [StoreFileCloser-130db0832f74edda61b4f19f117dbb07-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 840b366be1f149a6a0f25329a1515460 2023-08-10T06:23:12,558 DEBUG [StoreFileCloser-130db0832f74edda61b4f19f117dbb07-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6f9e71fc2fea41d49c5fddcb29e166d7 2023-08-10T06:23:12,560 DEBUG [StoreFileCloser-130db0832f74edda61b4f19f117dbb07-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: fd5f4ea6e4a040f0bdfced6ae21741da 2023-08-10T06:23:12,565 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/recovered.edits/27.seqid, newMaxSeqId=27, maxSeqId=1 2023-08-10T06:23:12,565 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:23:12,566 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1909): Closed backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:23:12,566 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1622): Region close journal for 130db0832f74edda61b4f19f117dbb07: 2023-08-10T06:23:12,569 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(163): Closed 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:12,569 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=39 updating hbase:meta row=130db0832f74edda61b4f19f117dbb07, regionState=CLOSED 2023-08-10T06:23:12,573 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=40, resume processing ppid=39 2023-08-10T06:23:12,573 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=40, ppid=39, state=SUCCESS; CloseRegionProcedure 130db0832f74edda61b4f19f117dbb07, server=jenkins-hbase3.apache.org,37733,1691648558732 in 1.0320 sec 2023-08-10T06:23:12,575 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=39, resume processing ppid=38 2023-08-10T06:23:12,576 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=39, ppid=38, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=130db0832f74edda61b4f19f117dbb07, UNASSIGN in 1.0390 sec 2023-08-10T06:23:12,576 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648592576"}]},"ts":"1691648592576"} 2023-08-10T06:23:12,578 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-08-10T06:23:12,580 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-08-10T06:23:12,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=38, state=SUCCESS; DisableTableProcedure table=backup:system in 1.0630 sec 2023-08-10T06:23:12,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-08-10T06:23:12,636 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] client.HBaseAdmin$TableFuture(3590): Operation: DISABLE, Table Name: backup:system, procId: 38 completed 2023-08-10T06:23:12,646 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] client.HBaseAdmin(2645): Taking restore-failsafe snapshot: hbase-failsafe-snapshot_backup_system-1691648592646 2023-08-10T06:23:12,647 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1691648592646 table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:23:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648592647 (current time:1691648592647). 2023-08-10T06:23:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:23:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1691648592646 VERSION not specified, setting to 2 2023-08-10T06:23:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x10d8bd66 to 127.0.0.1:61601 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:23:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@790aee1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:23:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:23:12,657 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:43230, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:23:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x10d8bd66 to 127.0.0.1:61601 2023-08-10T06:23:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:23:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:23:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure2.ProcedureExecutor(1033): Stored pid=41, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648592646 table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:23:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648592646 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 41 2023-08-10T06:23:12,662 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648592646 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:23:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-08-10T06:23:12,663 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648592646 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:23:12,665 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648592646 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:23:12,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-08-10T06:23:12,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-08-10T06:23:13,072 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648592646 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-08-10T06:23:13,074 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-08-10T06:23:13,074 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-08-10T06:23:13,074 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:23:13,076 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/37038161122946e9af290add645e1bb6, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/6d6621fec99f4115886e4e523164d414] hfiles 2023-08-10T06:23:13,076 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/2): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/37038161122946e9af290add645e1bb6 2023-08-10T06:23:13,076 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/2): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/6d6621fec99f4115886e4e523164d414 2023-08-10T06:23:13,076 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:23:13,078 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/6f9e71fc2fea41d49c5fddcb29e166d7, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/840b366be1f149a6a0f25329a1515460, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/fd5f4ea6e4a040f0bdfced6ae21741da] hfiles 2023-08-10T06:23:13,078 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/3): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/6f9e71fc2fea41d49c5fddcb29e166d7 2023-08-10T06:23:13,078 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/3): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/840b366be1f149a6a0f25329a1515460 2023-08-10T06:23:13,078 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/3): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/fd5f4ea6e4a040f0bdfced6ae21741da 2023-08-10T06:23:13,084 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 130db0832f74edda61b4f19f117dbb07, NAME => 'backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-08-10T06:23:13,085 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648592646 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:23:13,086 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648592646 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:23:13,086 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1691648592646 2023-08-10T06:23:13,087 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1691648592646 2023-08-10T06:23:13,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-08-10T06:23:13,497 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648592646 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:23:13,506 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648592646 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:23:13,507 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1691648592646 to hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1691648592646 2023-08-10T06:23:13,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-08-10T06:23:14,327 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648592646 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:23:14,327 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648592646 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 41 2023-08-10T06:23:14,328 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=41, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648592646 table=backup:system type=FLUSH ttl=0 } in 1.6680 sec 2023-08-10T06:23:14,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-08-10T06:23:14,769 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 41 completed 2023-08-10T06:23:14,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure2.ProcedureExecutor(1033): Stored pid=42, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691648587226 type: FLUSH version: 2 ttl: 0 ) 2023-08-10T06:23:14,797 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(1015): Restore snapshot=snapshot_backup_system as table=backup:system 2023-08-10T06:23:14,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-08-10T06:23:14,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-08-10T06:23:15,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-08-10T06:23:15,212 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/.tabledesc/.tableinfo.0000000001.635 2023-08-10T06:23:15,212 INFO [PEWorker-3 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2023-08-10T06:23:15,213 INFO [PEWorker-3 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:23:15,219 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691648587226 type: FLUSH version: 2 ttl: 0 2023-08-10T06:23:15,220 DEBUG [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system 2023-08-10T06:23:15,223 DEBUG [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-08-10T06:23:15,223 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(227): region to restore: 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:15,238 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/37038161122946e9af290add645e1bb6 to hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/archive/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/37038161122946e9af290add645e1bb6 2023-08-10T06:23:15,239 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:23:15,243 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/fd5f4ea6e4a040f0bdfced6ae21741da to hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/archive/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/fd5f4ea6e4a040f0bdfced6ae21741da 2023-08-10T06:23:15,243 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:23:15,243 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691648587226 type: FLUSH version: 2 ttl: 0 2023-08-10T06:23:15,244 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1691648595244"}]},"ts":"9223372036854775807"} 2023-08-10T06:23:15,247 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-08-10T06:23:15,247 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 130db0832f74edda61b4f19f117dbb07, NAME => 'backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:23:15,247 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1691648595245"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648595245"}]},"ts":"1691648595245"} 2023-08-10T06:23:15,249 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:23:15,249 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-08-10T06:23:15,249 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => 130db0832f74edda61b4f19f117dbb07, NAME => 'backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:23:15,249 INFO [PEWorker-3 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-08-10T06:23:15,252 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=42, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691648587226 type: FLUSH version: 2 ttl: 0 ) in 465 msec 2023-08-10T06:23:15,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-08-10T06:23:15,405 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] client.HBaseAdmin$TableFuture(3590): Operation: MODIFY, Table Name: backup:system, procId: 42 completed 2023-08-10T06:23:15,405 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] client.HBaseAdmin(2678): Deleting restore-failsafe snapshot: hbase-failsafe-snapshot_backup_system-1691648592646 2023-08-10T06:23:15,406 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1691648592646" 2023-08-10T06:23:15,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.MetricsHBaseServer(144): Unknown exception type java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1155) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:375) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:760) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-08-10T06:23:15,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.CallRunner(138): callId: 196 service: MasterService methodName: DeleteSnapshot size: 86 connection: 172.31.12.81:48964 deadline: 1691648655406, exception=java.io.IOException: preDeleteSnapshot 2023-08-10T06:23:15,514 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1691648592646" 2023-08-10T06:23:15,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.MetricsHBaseServer(144): Unknown exception type java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1155) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:375) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:760) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-08-10T06:23:15,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.CallRunner(138): callId: 198 service: MasterService methodName: DeleteSnapshot size: 86 connection: 172.31.12.81:48964 deadline: 1691648655512, exception=java.io.IOException: preDeleteSnapshot 2023-08-10T06:23:15,518 ERROR [Listener at jenkins-hbase3.apache.org/36469 {}] client.HBaseAdmin(2681): Unable to remove the failsafe snapshot: hbase-failsafe-snapshot_backup_system-1691648592646 org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2023-08-10T06:23:15.411Z, RpcRetryingCaller{globalStartTime=2023-08-10T06:23:15.406Z, pause=100, maxAttempts=2}, java.io.IOException: java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1155) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1152) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1152) at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:375) at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:760) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) 2023-08-10T06:23:15.518Z, RpcRetryingCaller{globalStartTime=2023-08-10T06:23:15.406Z, pause=100, maxAttempts=2}, java.io.IOException: java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1155) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1152) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1152) at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:375) at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:760) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:142) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3018) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3010) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.deleteSnapshot(HBaseAdmin.java:2918) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.restoreSnapshot(HBaseAdmin.java:2679) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.restoreSnapshot(HBaseAdmin.java:2617) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.restoreSnapshot(HBaseAdmin.java:2582) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.backup.impl.BackupSystemTable.restoreFromSnapshot(BackupSystemTable.java:1523) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.deleteBackups(BackupAdminImpl.java:155) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures.testBackupDeleteWithFailuresAfter(TestBackupDeleteWithFailures.java:161) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures.testBackupDeleteWithFailures(TestBackupDeleteWithFailures.java:136) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1155) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1152) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1152) at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:375) at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:760) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.MasterCallable.call(MasterCallable.java:101) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] ... 34 more Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1155) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1152) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1152) at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:375) at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:760) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:165) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:195) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] ... 1 more 2023-08-10T06:23:15,522 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] client.HBaseAdmin$17(903): Started enable of backup:system 2023-08-10T06:23:15,527 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.HMaster$11(2630): Client=jenkins//172.31.12.81 enable backup:system 2023-08-10T06:23:15,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure2.ProcedureExecutor(1033): Stored pid=43, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-08-10T06:23:15,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=43 2023-08-10T06:23:15,539 INFO [PEWorker-4 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-08-10T06:23:15,539 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648595539"}]},"ts":"1691648595539"} 2023-08-10T06:23:15,540 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-08-10T06:23:15,544 DEBUG [PEWorker-4 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-08-10T06:23:15,544 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=130db0832f74edda61b4f19f117dbb07, ASSIGN}] 2023-08-10T06:23:15,546 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=130db0832f74edda61b4f19f117dbb07, ASSIGN 2023-08-10T06:23:15,547 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=130db0832f74edda61b4f19f117dbb07, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-08-10T06:23:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=43 2023-08-10T06:23:15,698 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=44 updating hbase:meta row=130db0832f74edda61b4f19f117dbb07, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:15,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE; OpenRegionProcedure 130db0832f74edda61b4f19f117dbb07, server=jenkins-hbase3.apache.org,37733,1691648558732}] 2023-08-10T06:23:15,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=43 2023-08-10T06:23:15,856 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(130): Open backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:23:15,856 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7254): Opening region: {ENCODED => 130db0832f74edda61b4f19f117dbb07, NAME => 'backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:23:15,857 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:23:15,857 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:15,857 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(888): Instantiated backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:23:15,857 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7296): checking encryption for 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:15,857 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7299): checking classloading for 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:15,858 INFO [StoreOpener-130db0832f74edda61b4f19f117dbb07-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:15,859 INFO [StoreOpener-130db0832f74edda61b4f19f117dbb07-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 130db0832f74edda61b4f19f117dbb07 columnFamilyName meta 2023-08-10T06:23:15,859 DEBUG [StoreOpener-130db0832f74edda61b4f19f117dbb07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:23:15,864 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-08-10T06:23:15,867 DEBUG [StoreOpener-130db0832f74edda61b4f19f117dbb07-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/6d6621fec99f4115886e4e523164d414 2023-08-10T06:23:15,867 INFO [StoreOpener-130db0832f74edda61b4f19f117dbb07-1 {}] regionserver.HStore(324): Store=130db0832f74edda61b4f19f117dbb07/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:23:15,867 INFO [StoreOpener-130db0832f74edda61b4f19f117dbb07-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:15,868 INFO [StoreOpener-130db0832f74edda61b4f19f117dbb07-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 130db0832f74edda61b4f19f117dbb07 columnFamilyName session 2023-08-10T06:23:15,868 DEBUG [StoreOpener-130db0832f74edda61b4f19f117dbb07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:23:15,875 DEBUG [StoreOpener-130db0832f74edda61b4f19f117dbb07-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/6f9e71fc2fea41d49c5fddcb29e166d7 2023-08-10T06:23:15,879 DEBUG [StoreOpener-130db0832f74edda61b4f19f117dbb07-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/840b366be1f149a6a0f25329a1515460 2023-08-10T06:23:15,880 INFO [StoreOpener-130db0832f74edda61b4f19f117dbb07-1 {}] regionserver.HStore(324): Store=130db0832f74edda61b4f19f117dbb07/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:23:15,880 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:15,882 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:15,883 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-08-10T06:23:15,884 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1076): writing seq id for 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:15,885 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): Opened 130db0832f74edda61b4f19f117dbb07; next sequenceid=28; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11364170400, jitterRate=0.05837084352970123}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-08-10T06:23:15,886 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(992): Region open journal for 130db0832f74edda61b4f19f117dbb07: 2023-08-10T06:23:15,887 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07., pid=45, masterSystemTime=1691648595852 2023-08-10T06:23:15,888 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:23:15,888 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(158): Opened backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:23:15,889 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=44 updating hbase:meta row=130db0832f74edda61b4f19f117dbb07, regionState=OPEN, openSeqNum=28, regionLocation=jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:15,893 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=45, resume processing ppid=44 2023-08-10T06:23:15,893 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=45, ppid=44, state=SUCCESS; OpenRegionProcedure 130db0832f74edda61b4f19f117dbb07, server=jenkins-hbase3.apache.org,37733,1691648558732 in 191 msec 2023-08-10T06:23:15,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=44, resume processing ppid=43 2023-08-10T06:23:15,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=44, ppid=43, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=130db0832f74edda61b4f19f117dbb07, ASSIGN in 349 msec 2023-08-10T06:23:15,895 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648595895"}]},"ts":"1691648595895"} 2023-08-10T06:23:15,896 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-08-10T06:23:15,898 INFO [PEWorker-5 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-08-10T06:23:15,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=43, state=SUCCESS; EnableTableProcedure table=backup:system in 370 msec 2023-08-10T06:23:16,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=43 2023-08-10T06:23:16,143 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] client.HBaseAdmin$TableFuture(3590): Operation: ENABLE, Table Name: backup:system, procId: 43 completed 2023-08-10T06:23:16,143 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1525): Done restoring backup system table 2023-08-10T06:23:16,143 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:23:16,153 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:23:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.MetricsHBaseServer(144): Unknown exception type java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1155) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:375) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:760) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-08-10T06:23:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.CallRunner(138): callId: 212 service: MasterService methodName: DeleteSnapshot size: 57 connection: 172.31.12.81:48964 deadline: 1691648656152, exception=java.io.IOException: preDeleteSnapshot 2023-08-10T06:23:16,257 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:23:16,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.MetricsHBaseServer(144): Unknown exception type java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1155) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:375) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:760) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-08-10T06:23:16,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.CallRunner(138): callId: 214 service: MasterService methodName: DeleteSnapshot size: 57 connection: 172.31.12.81:48964 deadline: 1691648656257, exception=java.io.IOException: preDeleteSnapshot 2023-08-10T06:23:16,261 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:23:16,289 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391f7175, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:23:16,293 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a5098f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:23:16,294 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:23:16,295 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:40666, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:23:16,300 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:23:16,301 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:43234, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService REPAIR status: no failed sessions found. Checking failed delete backup operation ... Found failed DELETE operation for: backup_1691648574696 Running DELETE again ... 2023-08-10T06:23:16,317 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-08-10T06:23:16,325 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable backup:system 2023-08-10T06:23:16,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure2.ProcedureExecutor(1033): Stored pid=46, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-08-10T06:23:16,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-08-10T06:23:16,330 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648596330"}]},"ts":"1691648596330"} 2023-08-10T06:23:16,331 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-08-10T06:23:16,334 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-08-10T06:23:16,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=130db0832f74edda61b4f19f117dbb07, UNASSIGN}] 2023-08-10T06:23:16,336 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=47, ppid=46, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=130db0832f74edda61b4f19f117dbb07, UNASSIGN 2023-08-10T06:23:16,337 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=47 updating hbase:meta row=130db0832f74edda61b4f19f117dbb07, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:16,338 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE; CloseRegionProcedure 130db0832f74edda61b4f19f117dbb07, server=jenkins-hbase3.apache.org,37733,1691648558732}] 2023-08-10T06:23:16,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-08-10T06:23:16,491 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] handler.UnassignRegionHandler(119): Close 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:16,492 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1668): Closing 130db0832f74edda61b4f19f117dbb07, disabling compactions & flushes 2023-08-10T06:23:16,492 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1690): Closing region backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:23:16,492 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:23:16,492 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. after waiting 0 ms 2023-08-10T06:23:16,492 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:23:16,492 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(2819): Flushing 130db0832f74edda61b4f19f117dbb07 2/2 column families, dataSize=44 B heapSize=632 B 2023-08-10T06:23:16,507 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/session/30780b3e0abf4943b6289d0ecd6ef03e is 48, key is activesession:/session:c/1691648596262/Put/seqid=0 2023-08-10T06:23:16,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-08-10T06:23:16,912 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44 B at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/session/30780b3e0abf4943b6289d0ecd6ef03e 2023-08-10T06:23:16,918 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 30780b3e0abf4943b6289d0ecd6ef03e 2023-08-10T06:23:16,919 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/session/30780b3e0abf4943b6289d0ecd6ef03e as hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/30780b3e0abf4943b6289d0ecd6ef03e 2023-08-10T06:23:16,924 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/30780b3e0abf4943b6289d0ecd6ef03e, entries=1, sequenceid=31, filesize=4.9 K 2023-08-10T06:23:16,925 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(3022): Finished flush of dataSize ~44 B/44, heapSize ~360 B/360, currentSize=0 B/0 for 130db0832f74edda61b4f19f117dbb07 in 433ms, sequenceid=31, compaction requested=true 2023-08-10T06:23:16,927 DEBUG [StoreFileCloser-130db0832f74edda61b4f19f117dbb07-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6d6621fec99f4115886e4e523164d414 2023-08-10T06:23:16,929 DEBUG [StoreFileCloser-130db0832f74edda61b4f19f117dbb07-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 840b366be1f149a6a0f25329a1515460 2023-08-10T06:23:16,931 DEBUG [StoreFileCloser-130db0832f74edda61b4f19f117dbb07-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6f9e71fc2fea41d49c5fddcb29e166d7 2023-08-10T06:23:16,932 DEBUG [StoreFileCloser-130db0832f74edda61b4f19f117dbb07-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 30780b3e0abf4943b6289d0ecd6ef03e 2023-08-10T06:23:16,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-08-10T06:23:16,936 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/recovered.edits/34.seqid, newMaxSeqId=34, maxSeqId=27 2023-08-10T06:23:16,937 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:23:16,937 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1909): Closed backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:23:16,937 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1622): Region close journal for 130db0832f74edda61b4f19f117dbb07: 2023-08-10T06:23:16,939 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] handler.UnassignRegionHandler(163): Closed 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:16,939 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=47 updating hbase:meta row=130db0832f74edda61b4f19f117dbb07, regionState=CLOSED 2023-08-10T06:23:16,942 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=48, resume processing ppid=47 2023-08-10T06:23:16,943 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=48, ppid=47, state=SUCCESS; CloseRegionProcedure 130db0832f74edda61b4f19f117dbb07, server=jenkins-hbase3.apache.org,37733,1691648558732 in 603 msec 2023-08-10T06:23:16,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=47, resume processing ppid=46 2023-08-10T06:23:16,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=47, ppid=46, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=130db0832f74edda61b4f19f117dbb07, UNASSIGN in 608 msec 2023-08-10T06:23:16,945 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648596945"}]},"ts":"1691648596945"} 2023-08-10T06:23:16,946 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-08-10T06:23:16,948 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-08-10T06:23:16,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=46, state=SUCCESS; DisableTableProcedure table=backup:system in 623 msec 2023-08-10T06:23:17,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-08-10T06:23:17,449 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1691648597448 table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:23:17,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648597449 (current time:1691648597449). 2023-08-10T06:23:17,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:23:17,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1691648597448 VERSION not specified, setting to 2 2023-08-10T06:23:17,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4648dff3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:23:17,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:23:17,458 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:43238, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:23:17,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:23:17,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:23:17,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure2.ProcedureExecutor(1033): Stored pid=49, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648597448 table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:23:17,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648597448 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 49 2023-08-10T06:23:17,462 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648597448 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:23:17,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-08-10T06:23:17,463 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648597448 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:23:17,465 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648597448 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:23:17,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-08-10T06:23:17,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-08-10T06:23:17,871 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648597448 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-08-10T06:23:17,873 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-08-10T06:23:17,873 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-08-10T06:23:17,873 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:23:17,874 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/6d6621fec99f4115886e4e523164d414] hfiles 2023-08-10T06:23:17,874 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/1): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/6d6621fec99f4115886e4e523164d414 2023-08-10T06:23:17,875 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:23:17,876 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/30780b3e0abf4943b6289d0ecd6ef03e, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/6f9e71fc2fea41d49c5fddcb29e166d7, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/840b366be1f149a6a0f25329a1515460] hfiles 2023-08-10T06:23:17,876 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/3): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/30780b3e0abf4943b6289d0ecd6ef03e 2023-08-10T06:23:17,876 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/3): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/6f9e71fc2fea41d49c5fddcb29e166d7 2023-08-10T06:23:17,876 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/3): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/840b366be1f149a6a0f25329a1515460 2023-08-10T06:23:18,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-08-10T06:23:18,118 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:23:18,283 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 130db0832f74edda61b4f19f117dbb07, NAME => 'backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-08-10T06:23:18,284 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648597448 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:23:18,285 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648597448 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:23:18,285 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1691648597448 2023-08-10T06:23:18,286 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1691648597448 2023-08-10T06:23:18,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-08-10T06:23:18,697 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648597448 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:23:18,705 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648597448 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:23:18,705 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1691648597448 to hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1691648597448 2023-08-10T06:23:19,524 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648597448 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:23:19,524 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648597448 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 49 2023-08-10T06:23:19,526 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=49, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648597448 table=backup:system type=FLUSH ttl=0 } in 2.0650 sec 2023-08-10T06:23:19,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-08-10T06:23:19,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure2.ProcedureExecutor(1033): Stored pid=50, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691648587226 type: FLUSH version: 2 ttl: 0 ) 2023-08-10T06:23:19,581 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(1015): Restore snapshot=snapshot_backup_system as table=backup:system 2023-08-10T06:23:19,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-08-10T06:23:19,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-08-10T06:23:19,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-08-10T06:23:19,996 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2023-08-10T06:23:19,996 INFO [PEWorker-3 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/.tabledesc/.tableinfo.0000000003.635 2023-08-10T06:23:19,998 INFO [PEWorker-3 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:23:20,000 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691648587226 type: FLUSH version: 2 ttl: 0 2023-08-10T06:23:20,000 DEBUG [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system 2023-08-10T06:23:20,004 DEBUG [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-08-10T06:23:20,004 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(227): region to restore: 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:20,008 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:23:20,012 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/30780b3e0abf4943b6289d0ecd6ef03e to hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/archive/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/30780b3e0abf4943b6289d0ecd6ef03e 2023-08-10T06:23:20,012 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:23:20,013 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691648587226 type: FLUSH version: 2 ttl: 0 2023-08-10T06:23:20,014 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1691648600014"}]},"ts":"9223372036854775807"} 2023-08-10T06:23:20,019 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-08-10T06:23:20,019 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 130db0832f74edda61b4f19f117dbb07, NAME => 'backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:23:20,019 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1691648600015"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648600015"}]},"ts":"1691648600015"} 2023-08-10T06:23:20,021 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:23:20,021 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-08-10T06:23:20,021 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => 130db0832f74edda61b4f19f117dbb07, NAME => 'backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:23:20,021 INFO [PEWorker-3 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-08-10T06:23:20,024 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=50, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691648587226 type: FLUSH version: 2 ttl: 0 ) in 443 msec 2023-08-10T06:23:20,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-08-10T06:23:20,188 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1691648597448" 2023-08-10T06:23:20,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(378): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1691648597448 2023-08-10T06:23:20,193 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.HMaster$11(2630): Client=jenkins//172.31.12.81 enable backup:system 2023-08-10T06:23:20,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure2.ProcedureExecutor(1033): Stored pid=51, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-08-10T06:23:20,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=51 2023-08-10T06:23:20,197 INFO [PEWorker-4 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-08-10T06:23:20,198 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648600198"}]},"ts":"1691648600198"} 2023-08-10T06:23:20,199 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-08-10T06:23:20,203 DEBUG [PEWorker-4 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-08-10T06:23:20,203 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=130db0832f74edda61b4f19f117dbb07, ASSIGN}] 2023-08-10T06:23:20,205 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=130db0832f74edda61b4f19f117dbb07, ASSIGN 2023-08-10T06:23:20,205 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=130db0832f74edda61b4f19f117dbb07, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-08-10T06:23:20,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=51 2023-08-10T06:23:20,357 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=130db0832f74edda61b4f19f117dbb07, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:20,359 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; OpenRegionProcedure 130db0832f74edda61b4f19f117dbb07, server=jenkins-hbase3.apache.org,37733,1691648558732}] 2023-08-10T06:23:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=51 2023-08-10T06:23:20,515 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(130): Open backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:23:20,515 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7254): Opening region: {ENCODED => 130db0832f74edda61b4f19f117dbb07, NAME => 'backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:23:20,516 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:23:20,516 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:20,516 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(888): Instantiated backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:23:20,516 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7296): checking encryption for 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:20,516 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7299): checking classloading for 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:20,517 INFO [StoreOpener-130db0832f74edda61b4f19f117dbb07-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:20,518 INFO [StoreOpener-130db0832f74edda61b4f19f117dbb07-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 130db0832f74edda61b4f19f117dbb07 columnFamilyName meta 2023-08-10T06:23:20,518 DEBUG [StoreOpener-130db0832f74edda61b4f19f117dbb07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:23:20,525 DEBUG [StoreOpener-130db0832f74edda61b4f19f117dbb07-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/6d6621fec99f4115886e4e523164d414 2023-08-10T06:23:20,525 INFO [StoreOpener-130db0832f74edda61b4f19f117dbb07-1 {}] regionserver.HStore(324): Store=130db0832f74edda61b4f19f117dbb07/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:23:20,525 INFO [StoreOpener-130db0832f74edda61b4f19f117dbb07-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:20,526 INFO [StoreOpener-130db0832f74edda61b4f19f117dbb07-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 130db0832f74edda61b4f19f117dbb07 columnFamilyName session 2023-08-10T06:23:20,526 DEBUG [StoreOpener-130db0832f74edda61b4f19f117dbb07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:23:20,532 DEBUG [StoreOpener-130db0832f74edda61b4f19f117dbb07-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/6f9e71fc2fea41d49c5fddcb29e166d7 2023-08-10T06:23:20,537 DEBUG [StoreOpener-130db0832f74edda61b4f19f117dbb07-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/840b366be1f149a6a0f25329a1515460 2023-08-10T06:23:20,537 INFO [StoreOpener-130db0832f74edda61b4f19f117dbb07-1 {}] regionserver.HStore(324): Store=130db0832f74edda61b4f19f117dbb07/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:23:20,538 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:20,539 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:20,540 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-08-10T06:23:20,541 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1076): writing seq id for 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:20,542 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): Opened 130db0832f74edda61b4f19f117dbb07; next sequenceid=35; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11731396640, jitterRate=0.0925714522600174}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-08-10T06:23:20,543 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(992): Region open journal for 130db0832f74edda61b4f19f117dbb07: 2023-08-10T06:23:20,544 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07., pid=53, masterSystemTime=1691648600511 2023-08-10T06:23:20,546 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:23:20,546 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(158): Opened backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:23:20,546 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=130db0832f74edda61b4f19f117dbb07, regionState=OPEN, openSeqNum=35, regionLocation=jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:20,550 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=53, resume processing ppid=52 2023-08-10T06:23:20,550 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=53, ppid=52, state=SUCCESS; OpenRegionProcedure 130db0832f74edda61b4f19f117dbb07, server=jenkins-hbase3.apache.org,37733,1691648558732 in 189 msec 2023-08-10T06:23:20,552 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=52, resume processing ppid=51 2023-08-10T06:23:20,552 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=52, ppid=51, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=130db0832f74edda61b4f19f117dbb07, ASSIGN in 347 msec 2023-08-10T06:23:20,552 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648600552"}]},"ts":"1691648600552"} 2023-08-10T06:23:20,554 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-08-10T06:23:20,556 INFO [PEWorker-5 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-08-10T06:23:20,557 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=51, state=SUCCESS; EnableTableProcedure table=backup:system in 363 msec 2023-08-10T06:23:20,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=51 2023-08-10T06:23:20,800 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1525): Done restoring backup system table 2023-08-10T06:23:20,800 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:23:20,816 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:23:20,829 WARN [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupAdminImpl(128): Backup system table snapshot exists 2023-08-10T06:23:20,833 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupAdminImpl(227): Deleting backup backup_1691648574696 ... 2023-08-10T06:23:20,835 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] util.BackupUtils(443): Trying to cleanup up target dir : backup_1691648574696 2023-08-10T06:23:20,835 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] util.BackupUtils(458): No data has been found in hdfs://localhost:46823/backupUT/backup_1691648574696/default/test-1691648567409. 2023-08-10T06:23:20,836 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] util.BackupUtils(465): hdfs://localhost:46823/backupUT/backup_1691648574696/default is empty, remove it. 2023-08-10T06:23:20,841 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupAdminImpl(269): 0 bulk loaded files out of 0 were deleted 2023-08-10T06:23:20,841 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(433): deleted 0 original bulkload rows 2023-08-10T06:23:20,842 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupAdminImpl(276): Delete backup backup_1691648574696 completed. 2023-08-10T06:23:20,847 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1014): test-1691648567409 2023-08-10T06:23:20,849 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:23:20,857 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:23:20,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:23:20,859 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:23:20,859 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(637): Finish backup exclusive operation DELETE operation finished OK: backup_1691648574696 No failed backup MERGE operation found 2023-08-10T06:23:20,862 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:23:20,866 ERROR [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1559): Snapshot snapshot_backup_system does not exists 2023-08-10T06:23:20,867 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:23:20,867 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:23:20,870 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] backup.TestBackupDeleteWithFailures(143): test repair backup delete on a single table with data and failures POST_DELETE_SNAPSHOT_FAILURE 2023-08-10T06:23:20,877 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379441b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:23:20,880 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:23:20,882 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:59352, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:23:20,883 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:23:20,885 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:43296, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:23:20,898 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:23:20,905 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:23:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648600905 (current time:1691648600905). 2023-08-10T06:23:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:23:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:23:20,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205c070e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:23:20,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:23:20,914 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:59366, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:23:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:23:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:23:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure2.ProcedureExecutor(1033): Stored pid=54, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:23:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 54 2023-08-10T06:23:20,918 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:23:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-08-10T06:23:20,919 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:23:20,921 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:23:21,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-08-10T06:23:21,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-08-10T06:23:21,327 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:23:21,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE; SnapshotRegionProcedure 130db0832f74edda61b4f19f117dbb07}] 2023-08-10T06:23:21,329 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=55, ppid=54, state=RUNNABLE; SnapshotRegionProcedure 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:21,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37733 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=55 2023-08-10T06:23:21,481 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:23:21,482 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2819): Flushing 130db0832f74edda61b4f19f117dbb07 2/2 column families, dataSize=417 B heapSize=1.49 KB 2023-08-10T06:23:21,497 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/meta/1e7ba44d3ce0479a82c66ad8d16173e2 is 91, key is incrbackupset:hdfs://localhost:46823/backupUT/meta:test-1691648567409/1691648600848/Put/seqid=0 2023-08-10T06:23:21,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-08-10T06:23:21,902 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=184 B at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/meta/1e7ba44d3ce0479a82c66ad8d16173e2 2023-08-10T06:23:21,908 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1e7ba44d3ce0479a82c66ad8d16173e2 2023-08-10T06:23:21,908 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 1e7ba44d3ce0479a82c66ad8d16173e2 2023-08-10T06:23:21,921 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/session/a066d9ae88754eafb7320b716301a931 is 59, key is session:backup_1691648574696/session:/1691648600841/DeleteFamily/seqid=0 2023-08-10T06:23:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-08-10T06:23:22,326 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=233 B at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/session/a066d9ae88754eafb7320b716301a931 2023-08-10T06:23:22,332 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a066d9ae88754eafb7320b716301a931 2023-08-10T06:23:22,332 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a066d9ae88754eafb7320b716301a931 2023-08-10T06:23:22,333 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/meta/1e7ba44d3ce0479a82c66ad8d16173e2 as hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/1e7ba44d3ce0479a82c66ad8d16173e2 2023-08-10T06:23:22,339 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1e7ba44d3ce0479a82c66ad8d16173e2 2023-08-10T06:23:22,339 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/1e7ba44d3ce0479a82c66ad8d16173e2, entries=2, sequenceid=45, filesize=5.2 K 2023-08-10T06:23:22,340 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/session/a066d9ae88754eafb7320b716301a931 as hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/a066d9ae88754eafb7320b716301a931 2023-08-10T06:23:22,346 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a066d9ae88754eafb7320b716301a931 2023-08-10T06:23:22,346 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/a066d9ae88754eafb7320b716301a931, entries=2, sequenceid=45, filesize=5.2 K 2023-08-10T06:23:22,347 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(3022): Finished flush of dataSize ~417 B/417, heapSize ~1.46 KB/1496, currentSize=0 B/0 for 130db0832f74edda61b4f19f117dbb07 in 865ms, sequenceid=45, compaction requested=true 2023-08-10T06:23:22,347 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-08-10T06:23:22,348 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2520): Flush status journal for 130db0832f74edda61b4f19f117dbb07: 2023-08-10T06:23:22,348 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. for snapshot_backup_system completed. 2023-08-10T06:23:22,348 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:23:22,348 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:23:22,348 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/6d6621fec99f4115886e4e523164d414, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/1e7ba44d3ce0479a82c66ad8d16173e2] hfiles 2023-08-10T06:23:22,348 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/6d6621fec99f4115886e4e523164d414 for snapshot=snapshot_backup_system 2023-08-10T06:23:22,348 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/1e7ba44d3ce0479a82c66ad8d16173e2 for snapshot=snapshot_backup_system 2023-08-10T06:23:22,348 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/840b366be1f149a6a0f25329a1515460, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/6f9e71fc2fea41d49c5fddcb29e166d7, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/a066d9ae88754eafb7320b716301a931] hfiles 2023-08-10T06:23:22,348 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/840b366be1f149a6a0f25329a1515460 for snapshot=snapshot_backup_system 2023-08-10T06:23:22,348 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/6f9e71fc2fea41d49c5fddcb29e166d7 for snapshot=snapshot_backup_system 2023-08-10T06:23:22,348 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/a066d9ae88754eafb7320b716301a931 for snapshot=snapshot_backup_system 2023-08-10T06:23:22,524 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-08-10T06:23:22,524 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-08-10T06:23:22,762 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:23:22,762 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=55 2023-08-10T06:23:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.HMaster(3994): Remote procedure done, pid=55 2023-08-10T06:23:22,762 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:22,763 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=55, ppid=54, state=RUNNABLE; SnapshotRegionProcedure 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:22,766 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=55, resume processing ppid=54 2023-08-10T06:23:22,766 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:23:22,766 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=55, ppid=54, state=SUCCESS; SnapshotRegionProcedure 130db0832f74edda61b4f19f117dbb07 in 1.4370 sec 2023-08-10T06:23:22,766 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:23:22,767 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:23:22,767 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:23:22,768 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:23:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-08-10T06:23:23,185 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:23:23,194 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:23:23,194 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:23:24,015 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:23:24,015 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 54 2023-08-10T06:23:24,016 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=54, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0990 sec 2023-08-10T06:23:25,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-08-10T06:23:25,026 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] impl.TableBackupClient(115): Backup backup_1691648600877 started at 1691648605026. 2023-08-10T06:23:25,028 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.TableBackupClient(119): Backup session backup_1691648600877 has been started. 2023-08-10T06:23:25,029 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-08-10T06:23:25,030 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-08-10T06:23:25,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-08-10T06:23:25,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-08-10T06:23:25,031 INFO [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-08-10T06:23:25,031 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-08-10T06:23:25,032 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-08-10T06:23:25,032 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-08-10T06:23:25,032 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:23:25,034 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:25,034 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:23:25,034 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:23:25,034 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-08-10T06:23:25,034 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-08-10T06:23:25,035 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-08-10T06:23:25,035 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:23:25,035 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-08-10T06:23:25,035 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-08-10T06:23:25,035 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-08-10T06:23:25,035 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-08-10T06:23:25,035 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-08-10T06:23:25,035 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-08-10T06:23:25,036 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-08-10T06:23:25,036 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-08-10T06:23:25,036 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,37733,1691648558732' joining acquired barrier for procedure (rolllog) in zk 2023-08-10T06:23:25,039 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-08-10T06:23:25,039 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:25,039 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:23:25,039 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:23:25,039 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-08-10T06:23:25,039 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:23:25,039 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:23:25,040 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:23:25,040 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:25,040 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:23:25,041 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,37733,1691648558732' joining acquired barrier for procedure 'rolllog' on coordinator 2023-08-10T06:23:25,041 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@2923c11c[Count = 0] remaining members to acquire global barrier 2023-08-10T06:23:25,041 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-08-10T06:23:25,041 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-08-10T06:23:25,047 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-08-10T06:23:25,047 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-08-10T06:23:25,047 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-08-10T06:23:25,047 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-08-10T06:23:25,047 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-08-10T06:23:25,047 DEBUG [rs(jenkins-hbase3.apache.org,37733,1691648558732)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:25,047 INFO [rs(jenkins-hbase3.apache.org,37733,1691648558732)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1691648581385 highest: 1691648581385 on jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:25,047 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37733%2C1691648558732:(num 1691648581385) roll requested 2023-08-10T06:23:25,059 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.1691648605047, exclude list is [], retry=0 2023-08-10T06:23:25,064 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45175,DS-cb72d2f9-6999-4dc2-81ce-221dee4d930e,DISK] 2023-08-10T06:23:25,066 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.1691648581385 with entries=35, filesize=7.33 KB; new WAL /user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.1691648605047 2023-08-10T06:23:25,067 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:45175,DS-cb72d2f9-6999-4dc2-81ce-221dee4d930e,DISK]] 2023-08-10T06:23:25,067 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.1691648581385 is not closed yet, will try archiving it next time 2023-08-10T06:23:25,067 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37733%2C1691648558732.meta:.meta(num 1691648581432) roll requested 2023-08-10T06:23:25,069 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.1691648581385 not finished, retry = 0 2023-08-10T06:23:25,079 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.meta.1691648605067.meta, exclude list is [], retry=0 2023-08-10T06:23:25,082 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45175,DS-cb72d2f9-6999-4dc2-81ce-221dee4d930e,DISK] 2023-08-10T06:23:25,107 DEBUG [rs(jenkins-hbase3.apache.org,37733,1691648558732)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 59 2023-08-10T06:23:25,107 INFO [rs(jenkins-hbase3.apache.org,37733,1691648558732)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1691648605047 on jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:25,109 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.meta.1691648581432.meta with entries=20, filesize=4.92 KB; new WAL /user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.meta.1691648605067.meta 2023-08-10T06:23:25,109 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:45175,DS-cb72d2f9-6999-4dc2-81ce-221dee4d930e,DISK]] 2023-08-10T06:23:25,109 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.meta.1691648581432.meta is not closed yet, will try archiving it next time 2023-08-10T06:23:25,119 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-08-10T06:23:25,119 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,37733,1691648558732' in zk 2023-08-10T06:23:25,122 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-08-10T06:23:25,122 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:25,122 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:23:25,122 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-08-10T06:23:25,122 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:23:25,123 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:23:25,123 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:23:25,124 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:23:25,124 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:23:25,125 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:25,125 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:23:25,125 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:23:25,126 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:25,126 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,37733,1691648558732': 2023-08-10T06:23:25,126 INFO [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-08-10T06:23:25,126 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,37733,1691648558732' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-08-10T06:23:25,126 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-08-10T06:23:25,127 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-08-10T06:23:25,127 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-08-10T06:23:25,127 INFO [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-08-10T06:23:25,128 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-08-10T06:23:25,128 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-08-10T06:23:25,129 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:23:25,129 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:23:25,129 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:23:25,129 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:23:25,129 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:23:25,129 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:23:25,129 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:23:25,129 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:23:25,130 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:23:25,130 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:23:25,131 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:25,131 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:23:25,133 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:23:25,133 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:25,136 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:23:25,136 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:23:25,137 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:23:25,137 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-08-10T06:23:25,137 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-08-10T06:23:25,137 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:23:25,137 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:23:25,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-08-10T06:23:25,241 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1691648605241_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } 2023-08-10T06:23:25,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648605241 (current time:1691648605241). 2023-08-10T06:23:25,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:23:25,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1691648605241_default_test-1691648567409 VERSION not specified, setting to 2 2023-08-10T06:23:25,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74408071, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:23:25,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:23:25,253 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:59380, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:23:25,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:23:25,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:23:25,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure2.ProcedureExecutor(1033): Stored pid=56, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1691648605241_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } 2023-08-10T06:23:25,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1691648605241_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 }, snapshot procedure id = 56 2023-08-10T06:23:25,257 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1691648605241_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:23:25,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=56 2023-08-10T06:23:25,258 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1691648605241_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:23:25,260 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1691648605241_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:23:25,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=56 2023-08-10T06:23:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=56 2023-08-10T06:23:25,666 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1691648605241_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:23:25,666 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE; SnapshotRegionProcedure 89ac3e8e753d2c1222fc1445a7518dbe}] 2023-08-10T06:23:25,668 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=57, ppid=56, state=RUNNABLE; SnapshotRegionProcedure 89ac3e8e753d2c1222fc1445a7518dbe 2023-08-10T06:23:25,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37733 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=57 2023-08-10T06:23:25,820 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe. 2023-08-10T06:23:25,820 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HRegion(2520): Flush status journal for 89ac3e8e753d2c1222fc1445a7518dbe: 2023-08-10T06:23:25,820 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe. for snapshot_1691648605241_default_test-1691648567409 completed. 2023-08-10T06:23:25,820 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(240): Storing 'test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe.' region-info for snapshot=snapshot_1691648605241_default_test-1691648567409 2023-08-10T06:23:25,821 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:23:25,821 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/default/test-1691648567409/89ac3e8e753d2c1222fc1445a7518dbe/f/6ec84d0138b7498790d3e4f4273b9498] hfiles 2023-08-10T06:23:25,821 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/default/test-1691648567409/89ac3e8e753d2c1222fc1445a7518dbe/f/6ec84d0138b7498790d3e4f4273b9498 for snapshot=snapshot_1691648605241_default_test-1691648567409 2023-08-10T06:23:25,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=56 2023-08-10T06:23:26,019 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:23:26,068 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-08-10T06:23:26,068 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-08-10T06:23:26,226 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe. 2023-08-10T06:23:26,226 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=57 2023-08-10T06:23:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.HMaster(3994): Remote procedure done, pid=57 2023-08-10T06:23:26,227 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1691648605241_default_test-1691648567409 on region 89ac3e8e753d2c1222fc1445a7518dbe 2023-08-10T06:23:26,228 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=57, ppid=56, state=RUNNABLE; SnapshotRegionProcedure 89ac3e8e753d2c1222fc1445a7518dbe 2023-08-10T06:23:26,230 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=57, resume processing ppid=56 2023-08-10T06:23:26,231 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=57, ppid=56, state=SUCCESS; SnapshotRegionProcedure 89ac3e8e753d2c1222fc1445a7518dbe in 563 msec 2023-08-10T06:23:26,231 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1691648605241_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:23:26,231 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1691648605241_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:23:26,232 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1691648605241_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:23:26,232 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1691648605241_default_test-1691648567409 2023-08-10T06:23:26,233 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/.tmp/snapshot_1691648605241_default_test-1691648567409 2023-08-10T06:23:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=56 2023-08-10T06:23:26,642 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1691648605241_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:23:26,648 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1691648605241_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:23:26,649 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/.tmp/snapshot_1691648605241_default_test-1691648567409 to hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/snapshot_1691648605241_default_test-1691648567409 2023-08-10T06:23:27,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=56 2023-08-10T06:23:27,467 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1691648605241_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:23:27,467 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1691648605241_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 }, snapshot procedure id = 56 2023-08-10T06:23:27,468 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=56, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1691648605241_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } in 2.2120 sec 2023-08-10T06:23:29,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=56 2023-08-10T06:23:29,364 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1691648600877 2023-08-10T06:23:29,364 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-08-10T06:23:29,365 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-08-10T06:23:29,365 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1691648600877_test-1691648567409 2023-08-10T06:23:29,365 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1691648605241_default_test-1691648567409 to hdfs://localhost:46823/backupUT/backup_1691648600877/default/test-1691648567409/ 2023-08-10T06:23:29,365 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-08-10T06:23:29,389 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:46823, inputRoot=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322 2023-08-10T06:23:29,389 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,39523,1691648555675_819554214_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:46823/backupUT/backup_1691648600877/default/test-1691648567409, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:46823/backupUT/backup_1691648600877/default/test-1691648567409/.hbase-snapshot/.tmp/snapshot_1691648605241_default_test-1691648567409 2023-08-10T06:23:29,389 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:46823, inputRoot=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322, snapshotDir=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/snapshot_1691648605241_default_test-1691648567409. 2023-08-10T06:23:29,397 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/snapshot_1691648605241_default_test-1691648567409 to hdfs://localhost:46823/backupUT/backup_1691648600877/default/test-1691648567409/.hbase-snapshot/.tmp/snapshot_1691648605241_default_test-1691648567409 2023-08-10T06:23:29,812 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:29,812 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:29,813 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:29,813 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:30,780 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/hadoop-8795409709904827865.jar 2023-08-10T06:23:30,780 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:30,781 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:30,781 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:30,782 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:30,782 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:30,783 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:30,783 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:30,783 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:30,784 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:30,785 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:23:30,785 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:23:30,786 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:23:30,786 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:23:30,787 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:23:30,787 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:23:30,787 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:23:30,788 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:23:30,788 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:23:30,789 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:23:30,789 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:23:30,790 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:23:30,790 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:23:30,791 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:23:30,792 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:23:30,792 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:23:30,792 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:23:30,793 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:23:30,793 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:23:30,836 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1691648605241_default_test-1691648567409' hfile list 2023-08-10T06:23:30,839 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-08-10T06:23:30,911 WARN [Listener at jenkins-hbase3.apache.org/36469 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/mapreduce.cluster.local.dir/1691648610878/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:23:30,911 WARN [Listener at jenkins-hbase3.apache.org/36469 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/mapreduce.cluster.local.dir/1691648610878/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:23:30,978 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-08-10T06:23:31,003 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/default/test-1691648567409/89ac3e8e753d2c1222fc1445a7518dbe/f/6ec84d0138b7498790d3e4f4273b9498 output=hdfs://localhost:46823/backupUT/backup_1691648600877/default/test-1691648567409/archive/data/default/test-1691648567409/89ac3e8e753d2c1222fc1445a7518dbe/f/6ec84d0138b7498790d3e4f4273b9498 2023-08-10T06:23:31,003 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-08-10T06:23:31,947 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-08-10T06:23:31,948 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-08-10T06:23:31,955 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1691648605241_default_test-1691648567409 2023-08-10T06:23:31,955 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1691648605241_default_test-1691648567409 finished. 2023-08-10T06:23:31,955 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1014): test-1691648567409 2023-08-10T06:23:31,962 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691648600877 2023-08-10T06:23:31,962 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-08-10T06:23:32,368 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:46823/backupUT/backup_1691648600877/.backup.manifest 2023-08-10T06:23:32,368 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.TableBackupClient(358): Backup backup_1691648600877 finished: type=FULL,tablelist=test-1691648567409,targetRootDir=hdfs://localhost:46823/backupUT,startts=1691648605026,completets=1691648611962,bytescopied=0 2023-08-10T06:23:32,368 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-08-10T06:23:32,368 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1691648605241_default_test-1691648567409 2023-08-10T06:23:32,369 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1691648605241_default_test-1691648567409" 2023-08-10T06:23:32,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1691648605241_default_test-1691648567409 2023-08-10T06:23:32,373 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1691648605241_default_test-1691648567409 for backup backup_1691648600877 succeeded. 2023-08-10T06:23:32,373 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:23:32,380 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:23:32,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:23:32,383 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:23:32,385 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:23:32,390 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] impl.TableBackupClient(376): Backup backup_1691648600877 completed. 2023-08-10T06:23:32,390 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:23:32,403 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] backup.TestBackupDeleteWithFailures(147): backup complete 2023-08-10T06:23:32,427 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:23:32,438 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:23:32,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648612438 (current time:1691648612438). 2023-08-10T06:23:32,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:23:32,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:23:32,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ecfaaf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:23:32,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:23:32,446 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:45814, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:23:32,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:23:32,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:23:32,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure2.ProcedureExecutor(1033): Stored pid=58, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:23:32,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 58 2023-08-10T06:23:32,449 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:23:32,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=58 2023-08-10T06:23:32,450 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:23:32,452 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:23:32,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=58 2023-08-10T06:23:32,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=58 2023-08-10T06:23:32,859 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:23:32,859 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE; SnapshotRegionProcedure 130db0832f74edda61b4f19f117dbb07}] 2023-08-10T06:23:32,861 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=59, ppid=58, state=RUNNABLE; SnapshotRegionProcedure 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:33,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37733 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=59 2023-08-10T06:23:33,013 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:23:33,013 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegion(2819): Flushing 130db0832f74edda61b4f19f117dbb07 2/2 column families, dataSize=1.06 KB heapSize=2.22 KB 2023-08-10T06:23:33,017 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/meta/bc7b88341c1f4449b337245ba1560753 is 169, key is trslm:hdfs://localhost:46823/backupUT\x00test-1691648567409/meta:log-roll-map/1691648611958/Put/seqid=0 2023-08-10T06:23:33,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=58 2023-08-10T06:23:33,422 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=511 B at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/meta/bc7b88341c1f4449b337245ba1560753 2023-08-10T06:23:33,428 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: bc7b88341c1f4449b337245ba1560753 2023-08-10T06:23:33,432 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/session/ab1bb1e7cf6d4ca083d6ddaae454a35d is 310, key is session:backup_1691648600877/session:context/1691648612384/Put/seqid=0 2023-08-10T06:23:33,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=58 2023-08-10T06:23:33,837 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=579 B at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/session/ab1bb1e7cf6d4ca083d6ddaae454a35d 2023-08-10T06:23:33,842 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: ab1bb1e7cf6d4ca083d6ddaae454a35d 2023-08-10T06:23:33,843 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/meta/bc7b88341c1f4449b337245ba1560753 as hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/bc7b88341c1f4449b337245ba1560753 2023-08-10T06:23:33,849 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/bc7b88341c1f4449b337245ba1560753, entries=5, sequenceid=57, filesize=5.5 K 2023-08-10T06:23:33,850 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/session/ab1bb1e7cf6d4ca083d6ddaae454a35d as hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/ab1bb1e7cf6d4ca083d6ddaae454a35d 2023-08-10T06:23:33,855 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/ab1bb1e7cf6d4ca083d6ddaae454a35d, entries=2, sequenceid=57, filesize=5.3 K 2023-08-10T06:23:33,856 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegion(3022): Finished flush of dataSize ~1.06 KB/1090, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 130db0832f74edda61b4f19f117dbb07 in 843ms, sequenceid=57, compaction requested=true 2023-08-10T06:23:33,856 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegion(2520): Flush status journal for 130db0832f74edda61b4f19f117dbb07: 2023-08-10T06:23:33,856 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. for snapshot_backup_system completed. 2023-08-10T06:23:33,856 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:23:33,856 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:23:33,856 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/6d6621fec99f4115886e4e523164d414, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/1e7ba44d3ce0479a82c66ad8d16173e2, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/bc7b88341c1f4449b337245ba1560753] hfiles 2023-08-10T06:23:33,856 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/6d6621fec99f4115886e4e523164d414 for snapshot=snapshot_backup_system 2023-08-10T06:23:33,856 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/1e7ba44d3ce0479a82c66ad8d16173e2 for snapshot=snapshot_backup_system 2023-08-10T06:23:33,856 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/bc7b88341c1f4449b337245ba1560753 for snapshot=snapshot_backup_system 2023-08-10T06:23:33,856 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/840b366be1f149a6a0f25329a1515460, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/6f9e71fc2fea41d49c5fddcb29e166d7, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/a066d9ae88754eafb7320b716301a931, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/ab1bb1e7cf6d4ca083d6ddaae454a35d] hfiles 2023-08-10T06:23:33,856 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/840b366be1f149a6a0f25329a1515460 for snapshot=snapshot_backup_system 2023-08-10T06:23:33,856 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/6f9e71fc2fea41d49c5fddcb29e166d7 for snapshot=snapshot_backup_system 2023-08-10T06:23:33,856 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/a066d9ae88754eafb7320b716301a931 for snapshot=snapshot_backup_system 2023-08-10T06:23:33,856 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/ab1bb1e7cf6d4ca083d6ddaae454a35d for snapshot=snapshot_backup_system 2023-08-10T06:23:34,264 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:23:34,264 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=59 2023-08-10T06:23:34,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.HMaster(3994): Remote procedure done, pid=59 2023-08-10T06:23:34,265 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:34,266 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=59, ppid=58, state=RUNNABLE; SnapshotRegionProcedure 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:34,268 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=59, resume processing ppid=58 2023-08-10T06:23:34,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=59, ppid=58, state=SUCCESS; SnapshotRegionProcedure 130db0832f74edda61b4f19f117dbb07 in 1.4080 sec 2023-08-10T06:23:34,269 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:23:34,269 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:23:34,270 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:23:34,270 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:23:34,271 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:23:34,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=58 2023-08-10T06:23:34,680 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:23:34,690 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:23:34,690 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:23:35,511 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:23:35,511 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 58 2023-08-10T06:23:35,512 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=58, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0640 sec 2023-08-10T06:23:36,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=58 2023-08-10T06:23:36,563 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupAdminImpl(227): Deleting backup backup_1691648600877 ... 2023-08-10T06:23:36,564 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] util.BackupUtils(435): Delete log files: .backup.manifest 2023-08-10T06:23:36,564 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] util.BackupUtils(435): Delete log files: default 2023-08-10T06:23:36,565 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] util.BackupUtils(443): Trying to cleanup up target dir : backup_1691648600877 2023-08-10T06:23:36,566 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] util.BackupUtils(458): No data has been found in hdfs://localhost:46823/backupUT/backup_1691648600877/default/test-1691648567409. 2023-08-10T06:23:36,566 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] util.BackupUtils(465): hdfs://localhost:46823/backupUT/backup_1691648600877/default is empty, remove it. 2023-08-10T06:23:36,570 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupAdminImpl(269): 0 bulk loaded files out of 0 were deleted 2023-08-10T06:23:36,570 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(433): deleted 0 original bulkload rows 2023-08-10T06:23:36,572 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupAdminImpl(276): Delete backup backup_1691648600877 completed. 2023-08-10T06:23:36,580 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1014): test-1691648567409 2023-08-10T06:23:36,582 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:23:36,590 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:23:36,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:23:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.MetricsHBaseServer(144): Unknown exception type java.io.IOException: postDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.postDeleteSnapshot(TestBackupDeleteWithFailures.java:111) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$89.call(MasterCoprocessorHost.java:1164) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$89.call(MasterCoprocessorHost.java:1161) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.postDeleteSnapshot(MasterCoprocessorHost.java:1161) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:386) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:760) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-08-10T06:23:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.CallRunner(138): callId: 286 service: MasterService methodName: DeleteSnapshot size: 57 connection: 172.31.12.81:48964 deadline: 1691648676589, exception=java.io.IOException: postDeleteSnapshot 2023-08-10T06:23:36,694 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:23:36,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.MetricsHBaseServer(144): Unknown exception type org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException: Snapshot 'snapshot_backup_system' doesn't exist on the filesystem at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:359) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:760) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-08-10T06:23:36,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.CallRunner(138): callId: 288 service: MasterService methodName: DeleteSnapshot size: 57 connection: 172.31.12.81:48964 deadline: 1691648676694, exception=org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException: Snapshot 'snapshot_backup_system' doesn't exist on the filesystem 2023-08-10T06:23:36,700 WARN [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupAdminImpl(163): Delete operation succeeded, there were some errors: org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException: org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException: Snapshot 'snapshot_backup_system' doesn't exist on the filesystem at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:359) at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:760) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.MasterCallable.call(MasterCallable.java:101) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3018) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3010) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.deleteSnapshot(HBaseAdmin.java:2918) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.backup.impl.BackupSystemTable.deleteSnapshot(BackupSystemTable.java:1556) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.deleteBackups(BackupAdminImpl.java:149) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures.testBackupDeleteWithFailuresAfter(TestBackupDeleteWithFailures.java:161) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures.testBackupDeleteWithFailures(TestBackupDeleteWithFailures.java:137) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException: Snapshot 'snapshot_backup_system' doesn't exist on the filesystem at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:359) at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:760) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:165) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:195) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] ... 1 more 2023-08-10T06:23:36,702 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:23:36,715 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5979cb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:23:36,719 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54b67eb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:23:36,720 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:23:36,722 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:37370, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:23:36,725 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:23:36,726 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:45830, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService REPAIR status: no failed sessions found. Checking failed delete backup operation ... No failed backup DELETE operation found 2023-08-10T06:23:36,741 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:23:36,745 ERROR [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1559): Snapshot snapshot_backup_system does not exists No failed backup MERGE operation found 2023-08-10T06:23:36,746 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:23:36,750 ERROR [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1559): Snapshot snapshot_backup_system does not exists 2023-08-10T06:23:36,750 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:23:36,750 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:23:36,757 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] backup.TestBackupDeleteWithFailures(143): test repair backup delete on a single table with data and failures PRE_SNAPSHOT_FAILURE 2023-08-10T06:23:36,770 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bcb8967, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:23:36,773 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:23:36,775 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:45832, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:23:36,776 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:23:36,777 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:37374, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:23:36,788 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:23:36,794 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:23:36,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648616794 (current time:1691648616794). 2023-08-10T06:23:36,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:23:36,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:23:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2433604f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:23:36,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:23:36,802 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:45846, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:23:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:23:36,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:23:36,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure2.ProcedureExecutor(1033): Stored pid=60, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:23:36,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2023-08-10T06:23:36,806 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:23:36,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-08-10T06:23:36,807 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:23:36,809 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:23:36,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-08-10T06:23:37,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-08-10T06:23:37,215 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:23:37,215 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE; SnapshotRegionProcedure 130db0832f74edda61b4f19f117dbb07}] 2023-08-10T06:23:37,217 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=61, ppid=60, state=RUNNABLE; SnapshotRegionProcedure 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:37,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37733 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=61 2023-08-10T06:23:37,369 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:23:37,369 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2819): Flushing 130db0832f74edda61b4f19f117dbb07 2/2 column families, dataSize=268 B heapSize=1.13 KB 2023-08-10T06:23:37,373 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/meta/287d80b85c894371951b5e08d8d1fa60 is 91, key is incrbackupset:hdfs://localhost:46823/backupUT/meta:test-1691648567409/1691648616580/Put/seqid=0 2023-08-10T06:23:37,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-08-10T06:23:37,778 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=124 B at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/meta/287d80b85c894371951b5e08d8d1fa60 2023-08-10T06:23:37,783 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 287d80b85c894371951b5e08d8d1fa60 2023-08-10T06:23:37,784 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 287d80b85c894371951b5e08d8d1fa60 2023-08-10T06:23:37,787 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/session/3215b68d03e9468cbb851950cf65b0d4 is 59, key is session:backup_1691648600877/session:/1691648616571/DeleteFamily/seqid=0 2023-08-10T06:23:37,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-08-10T06:23:38,191 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=144 B at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/session/3215b68d03e9468cbb851950cf65b0d4 2023-08-10T06:23:38,197 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3215b68d03e9468cbb851950cf65b0d4 2023-08-10T06:23:38,197 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3215b68d03e9468cbb851950cf65b0d4 2023-08-10T06:23:38,198 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/meta/287d80b85c894371951b5e08d8d1fa60 as hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/287d80b85c894371951b5e08d8d1fa60 2023-08-10T06:23:38,203 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 287d80b85c894371951b5e08d8d1fa60 2023-08-10T06:23:38,203 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/287d80b85c894371951b5e08d8d1fa60, entries=2, sequenceid=65, filesize=5.2 K 2023-08-10T06:23:38,204 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/session/3215b68d03e9468cbb851950cf65b0d4 as hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/3215b68d03e9468cbb851950cf65b0d4 2023-08-10T06:23:38,210 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3215b68d03e9468cbb851950cf65b0d4 2023-08-10T06:23:38,210 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/3215b68d03e9468cbb851950cf65b0d4, entries=2, sequenceid=65, filesize=5.2 K 2023-08-10T06:23:38,211 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(3022): Finished flush of dataSize ~268 B/268, heapSize ~1.09 KB/1120, currentSize=0 B/0 for 130db0832f74edda61b4f19f117dbb07 in 842ms, sequenceid=65, compaction requested=true 2023-08-10T06:23:38,211 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2520): Flush status journal for 130db0832f74edda61b4f19f117dbb07: 2023-08-10T06:23:38,211 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. for snapshot_backup_system completed. 2023-08-10T06:23:38,211 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:23:38,212 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:23:38,212 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/6d6621fec99f4115886e4e523164d414, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/1e7ba44d3ce0479a82c66ad8d16173e2, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/bc7b88341c1f4449b337245ba1560753, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/287d80b85c894371951b5e08d8d1fa60] hfiles 2023-08-10T06:23:38,212 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/6d6621fec99f4115886e4e523164d414 for snapshot=snapshot_backup_system 2023-08-10T06:23:38,212 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/1e7ba44d3ce0479a82c66ad8d16173e2 for snapshot=snapshot_backup_system 2023-08-10T06:23:38,212 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/bc7b88341c1f4449b337245ba1560753 for snapshot=snapshot_backup_system 2023-08-10T06:23:38,212 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/287d80b85c894371951b5e08d8d1fa60 for snapshot=snapshot_backup_system 2023-08-10T06:23:38,212 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/840b366be1f149a6a0f25329a1515460, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/6f9e71fc2fea41d49c5fddcb29e166d7, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/a066d9ae88754eafb7320b716301a931, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/ab1bb1e7cf6d4ca083d6ddaae454a35d, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/3215b68d03e9468cbb851950cf65b0d4] hfiles 2023-08-10T06:23:38,212 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/840b366be1f149a6a0f25329a1515460 for snapshot=snapshot_backup_system 2023-08-10T06:23:38,212 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/6f9e71fc2fea41d49c5fddcb29e166d7 for snapshot=snapshot_backup_system 2023-08-10T06:23:38,212 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/a066d9ae88754eafb7320b716301a931 for snapshot=snapshot_backup_system 2023-08-10T06:23:38,212 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/ab1bb1e7cf6d4ca083d6ddaae454a35d for snapshot=snapshot_backup_system 2023-08-10T06:23:38,212 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/3215b68d03e9468cbb851950cf65b0d4 for snapshot=snapshot_backup_system 2023-08-10T06:23:38,618 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:23:38,618 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=61 2023-08-10T06:23:38,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.HMaster(3994): Remote procedure done, pid=61 2023-08-10T06:23:38,619 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:38,620 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=61, ppid=60, state=RUNNABLE; SnapshotRegionProcedure 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:38,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=61, resume processing ppid=60 2023-08-10T06:23:38,622 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:23:38,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=61, ppid=60, state=SUCCESS; SnapshotRegionProcedure 130db0832f74edda61b4f19f117dbb07 in 1.4060 sec 2023-08-10T06:23:38,623 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:23:38,623 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:23:38,623 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:23:38,624 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:23:38,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-08-10T06:23:39,035 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:23:39,049 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:23:39,049 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:23:39,873 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:23:39,873 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2023-08-10T06:23:39,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=60, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0690 sec 2023-08-10T06:23:40,541 DEBUG [regionserver/jenkins-hbase3:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.15384615384615385, tune throughput to 57.69 MB/second 2023-08-10T06:23:40,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-08-10T06:23:40,914 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] impl.TableBackupClient(115): Backup backup_1691648616770 started at 1691648620914. 2023-08-10T06:23:40,916 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.TableBackupClient(119): Backup session backup_1691648616770 has been started. 2023-08-10T06:23:40,918 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-08-10T06:23:40,919 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-08-10T06:23:40,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-08-10T06:23:40,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-08-10T06:23:40,919 INFO [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-08-10T06:23:40,919 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-08-10T06:23:40,920 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-08-10T06:23:40,920 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-08-10T06:23:40,921 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:23:40,924 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:40,924 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:23:40,924 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:23:40,924 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-08-10T06:23:40,924 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-08-10T06:23:40,925 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-08-10T06:23:40,925 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:23:40,925 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-08-10T06:23:40,925 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-08-10T06:23:40,925 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-08-10T06:23:40,925 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-08-10T06:23:40,925 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-08-10T06:23:40,925 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-08-10T06:23:40,926 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-08-10T06:23:40,926 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-08-10T06:23:40,926 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,37733,1691648558732' joining acquired barrier for procedure (rolllog) in zk 2023-08-10T06:23:40,929 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-08-10T06:23:40,929 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:40,929 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:23:40,929 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:23:40,929 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-08-10T06:23:40,929 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:23:40,930 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:23:40,930 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:23:40,930 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:40,930 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:23:40,931 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,37733,1691648558732' joining acquired barrier for procedure 'rolllog' on coordinator 2023-08-10T06:23:40,931 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@3700b390[Count = 0] remaining members to acquire global barrier 2023-08-10T06:23:40,931 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-08-10T06:23:40,931 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-08-10T06:23:40,932 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-08-10T06:23:40,932 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-08-10T06:23:40,932 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-08-10T06:23:40,933 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-08-10T06:23:40,933 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-08-10T06:23:40,933 DEBUG [rs(jenkins-hbase3.apache.org,37733,1691648558732)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:40,933 INFO [rs(jenkins-hbase3.apache.org,37733,1691648558732)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1691648605047 highest: 1691648605047 on jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:40,933 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37733%2C1691648558732:(num 1691648605047) roll requested 2023-08-10T06:23:40,944 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.1691648620933, exclude list is [], retry=0 2023-08-10T06:23:40,949 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45175,DS-cb72d2f9-6999-4dc2-81ce-221dee4d930e,DISK] 2023-08-10T06:23:40,952 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.1691648605047 with entries=18, filesize=3.44 KB; new WAL /user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.1691648620933 2023-08-10T06:23:40,952 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:45175,DS-cb72d2f9-6999-4dc2-81ce-221dee4d930e,DISK]] 2023-08-10T06:23:40,952 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.1691648605047 is not closed yet, will try archiving it next time 2023-08-10T06:23:40,954 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37733%2C1691648558732.meta:.meta(num 1691648605067) roll requested 2023-08-10T06:23:40,954 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.1691648581385 to hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/oldWALs/jenkins-hbase3.apache.org%2C37733%2C1691648558732.1691648581385 2023-08-10T06:23:40,970 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.meta.1691648620954.meta, exclude list is [], retry=0 2023-08-10T06:23:40,974 DEBUG [rs(jenkins-hbase3.apache.org,37733,1691648558732)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 41 2023-08-10T06:23:40,974 INFO [rs(jenkins-hbase3.apache.org,37733,1691648558732)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1691648620933 on jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:40,975 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45175,DS-cb72d2f9-6999-4dc2-81ce-221dee4d930e,DISK] 2023-08-10T06:23:40,977 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.meta.1691648605067.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.meta.1691648620954.meta 2023-08-10T06:23:40,977 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:45175,DS-cb72d2f9-6999-4dc2-81ce-221dee4d930e,DISK]] 2023-08-10T06:23:40,977 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.meta.1691648605067.meta is not closed yet, will try archiving it next time 2023-08-10T06:23:40,980 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.meta.1691648605067.meta not finished, retry = 0 2023-08-10T06:23:40,987 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-08-10T06:23:40,987 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,37733,1691648558732' in zk 2023-08-10T06:23:40,989 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-08-10T06:23:40,989 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:23:40,989 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:40,989 DEBUG [member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-08-10T06:23:40,990 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:23:40,990 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:23:40,991 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:23:40,991 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:23:40,991 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:23:40,991 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:40,992 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:23:40,992 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:23:40,993 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:40,993 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,37733,1691648558732': 2023-08-10T06:23:40,993 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,37733,1691648558732' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-08-10T06:23:40,993 INFO [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-08-10T06:23:40,993 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-08-10T06:23:40,993 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-08-10T06:23:40,993 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-08-10T06:23:40,993 INFO [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-08-10T06:23:40,996 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-08-10T06:23:40,996 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-08-10T06:23:40,996 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:23:40,996 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:23:40,996 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:23:40,997 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:23:40,997 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:23:40,997 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:23:40,997 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:23:40,997 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:23:40,998 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:23:40,998 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:23:41,004 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:41,005 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:23:41,005 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:23:41,006 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:41,009 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:23:41,009 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:23:41,010 DEBUG [(jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:23:41,010 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-08-10T06:23:41,011 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-08-10T06:23:41,011 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:23:41,011 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:23:41,081 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.meta.1691648605067.meta to hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/oldWALs/jenkins-hbase3.apache.org%2C37733%2C1691648558732.meta.1691648605067.meta 2023-08-10T06:23:41,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-08-10T06:23:41,115 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1691648621114_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } 2023-08-10T06:23:41,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648621115 (current time:1691648621115). 2023-08-10T06:23:41,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:23:41,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1691648621114_default_test-1691648567409 VERSION not specified, setting to 2 2023-08-10T06:23:41,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25e49738, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:23:41,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:23:41,124 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56336, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:23:41,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:23:41,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:23:41,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure2.ProcedureExecutor(1033): Stored pid=62, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1691648621114_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } 2023-08-10T06:23:41,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1691648621114_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-08-10T06:23:41,127 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1691648621114_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:23:41,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-08-10T06:23:41,128 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1691648621114_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:23:41,130 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1691648621114_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:23:41,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-08-10T06:23:41,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-08-10T06:23:41,494 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 39c91a27ba70e64360f6a7e105d4cca0 changed from -1.0 to 0.0, refreshing cache 2023-08-10T06:23:41,538 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1691648621114_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:23:41,538 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure 89ac3e8e753d2c1222fc1445a7518dbe}] 2023-08-10T06:23:41,542 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure 89ac3e8e753d2c1222fc1445a7518dbe 2023-08-10T06:23:41,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37733 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=63 2023-08-10T06:23:41,694 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe. 2023-08-10T06:23:41,694 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegion(2520): Flush status journal for 89ac3e8e753d2c1222fc1445a7518dbe: 2023-08-10T06:23:41,694 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe. for snapshot_1691648621114_default_test-1691648567409 completed. 2023-08-10T06:23:41,694 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(240): Storing 'test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe.' region-info for snapshot=snapshot_1691648621114_default_test-1691648567409 2023-08-10T06:23:41,694 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:23:41,694 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/default/test-1691648567409/89ac3e8e753d2c1222fc1445a7518dbe/f/6ec84d0138b7498790d3e4f4273b9498] hfiles 2023-08-10T06:23:41,694 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/default/test-1691648567409/89ac3e8e753d2c1222fc1445a7518dbe/f/6ec84d0138b7498790d3e4f4273b9498 for snapshot=snapshot_1691648621114_default_test-1691648567409 2023-08-10T06:23:41,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-08-10T06:23:42,101 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe. 2023-08-10T06:23:42,101 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=63 2023-08-10T06:23:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.HMaster(3994): Remote procedure done, pid=63 2023-08-10T06:23:42,101 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1691648621114_default_test-1691648567409 on region 89ac3e8e753d2c1222fc1445a7518dbe 2023-08-10T06:23:42,104 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure 89ac3e8e753d2c1222fc1445a7518dbe 2023-08-10T06:23:42,106 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=63, resume processing ppid=62 2023-08-10T06:23:42,107 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=63, ppid=62, state=SUCCESS; SnapshotRegionProcedure 89ac3e8e753d2c1222fc1445a7518dbe in 567 msec 2023-08-10T06:23:42,107 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1691648621114_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:23:42,107 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1691648621114_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:23:42,108 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1691648621114_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:23:42,108 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1691648621114_default_test-1691648567409 2023-08-10T06:23:42,109 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/.tmp/snapshot_1691648621114_default_test-1691648567409 2023-08-10T06:23:42,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-08-10T06:23:42,520 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1691648621114_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:23:42,528 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1691648621114_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:23:42,528 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/.tmp/snapshot_1691648621114_default_test-1691648567409 to hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/snapshot_1691648621114_default_test-1691648567409 2023-08-10T06:23:43,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-08-10T06:23:43,348 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1691648621114_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:23:43,348 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1691648621114_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-08-10T06:23:43,349 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=62, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1691648621114_default_test-1691648567409 table=test-1691648567409 type=FLUSH ttl=0 } in 2.2230 sec 2023-08-10T06:23:45,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-08-10T06:23:45,234 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1691648616770 2023-08-10T06:23:45,235 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-08-10T06:23:45,235 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-08-10T06:23:45,235 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1691648616770_test-1691648567409 2023-08-10T06:23:45,235 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1691648621114_default_test-1691648567409 to hdfs://localhost:46823/backupUT/backup_1691648616770/default/test-1691648567409/ 2023-08-10T06:23:45,235 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-08-10T06:23:45,268 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:46823, inputRoot=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322 2023-08-10T06:23:45,268 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,39523,1691648555675_-1877584481_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:46823/backupUT/backup_1691648616770/default/test-1691648567409, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:46823/backupUT/backup_1691648616770/default/test-1691648567409/.hbase-snapshot/.tmp/snapshot_1691648621114_default_test-1691648567409 2023-08-10T06:23:45,268 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:46823, inputRoot=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322, snapshotDir=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/snapshot_1691648621114_default_test-1691648567409. 2023-08-10T06:23:45,279 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/snapshot_1691648621114_default_test-1691648567409 to hdfs://localhost:46823/backupUT/backup_1691648616770/default/test-1691648567409/.hbase-snapshot/.tmp/snapshot_1691648621114_default_test-1691648567409 2023-08-10T06:23:45,700 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:45,700 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:45,701 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:45,701 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:46,716 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/hadoop-6108199111144177451.jar 2023-08-10T06:23:46,716 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:46,716 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:46,717 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:46,717 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:46,718 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:46,718 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:46,719 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:46,719 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:46,719 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:23:46,720 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:23:46,721 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:23:46,721 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:23:46,722 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:23:46,722 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:23:46,723 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:23:46,724 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:23:46,724 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:23:46,725 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:23:46,725 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:23:46,726 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:23:46,726 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:23:46,727 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:23:46,728 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:23:46,729 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:23:46,729 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:23:46,729 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:23:46,730 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:23:46,730 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:23:46,775 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1691648621114_default_test-1691648567409' hfile list 2023-08-10T06:23:46,777 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-08-10T06:23:46,864 WARN [Listener at jenkins-hbase3.apache.org/36469 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/mapreduce.cluster.local.dir/1691648626824/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:23:46,864 WARN [Listener at jenkins-hbase3.apache.org/36469 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/mapreduce.cluster.local.dir/1691648626824/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:23:46,918 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-08-10T06:23:46,970 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/default/test-1691648567409/89ac3e8e753d2c1222fc1445a7518dbe/f/6ec84d0138b7498790d3e4f4273b9498 output=hdfs://localhost:46823/backupUT/backup_1691648616770/default/test-1691648567409/archive/data/default/test-1691648567409/89ac3e8e753d2c1222fc1445a7518dbe/f/6ec84d0138b7498790d3e4f4273b9498 2023-08-10T06:23:46,971 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 4.004M/sec 2023-08-10T06:23:47,900 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-08-10T06:23:47,902 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-08-10T06:23:47,909 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1691648621114_default_test-1691648567409 2023-08-10T06:23:47,909 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1691648621114_default_test-1691648567409 finished. 2023-08-10T06:23:47,909 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1014): test-1691648567409 2023-08-10T06:23:47,915 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691648616770 2023-08-10T06:23:47,915 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-08-10T06:23:48,321 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:46823/backupUT/backup_1691648616770/.backup.manifest 2023-08-10T06:23:48,321 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.TableBackupClient(358): Backup backup_1691648616770 finished: type=FULL,tablelist=test-1691648567409,targetRootDir=hdfs://localhost:46823/backupUT,startts=1691648620914,completets=1691648627915,bytescopied=0 2023-08-10T06:23:48,321 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-08-10T06:23:48,321 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1691648621114_default_test-1691648567409 2023-08-10T06:23:48,322 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1691648621114_default_test-1691648567409" 2023-08-10T06:23:48,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1691648621114_default_test-1691648567409 2023-08-10T06:23:48,327 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1691648621114_default_test-1691648567409 for backup backup_1691648616770 succeeded. 2023-08-10T06:23:48,329 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:23:48,335 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:23:48,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:23:48,338 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:23:48,348 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:23:48,354 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] impl.TableBackupClient(376): Backup backup_1691648616770 completed. 2023-08-10T06:23:48,355 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:23:48,368 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] backup.TestBackupDeleteWithFailures(147): backup complete 2023-08-10T06:23:48,391 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:23:48,403 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:23:48,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648628403 (current time:1691648628403). 2023-08-10T06:23:48,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:23:48,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:23:48,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c5c615, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:23:48,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:23:48,414 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56352, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:23:48,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:23:48,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:23:48,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure2.ProcedureExecutor(1033): Stored pid=64, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:23:48,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-08-10T06:23:48,419 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:23:48,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-08-10T06:23:48,420 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:23:48,422 WARN [PEWorker-1 {}] procedure.SnapshotProcedure(200): unexpected exception while execute pid=64, state=FAILED:SNAPSHOT_PRE_OPERATION, locked=true, exception=java.io.IOException via master-snapshot:java.io.IOException: preSnapshot; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }. Mark procedure Failed. java.io.IOException: preSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preSnapshot(TestBackupDeleteWithFailures.java:95) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$79.call(MasterCoprocessorHost.java:1068) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$79.call(MasterCoprocessorHost.java:1065) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preSnapshot(MasterCoprocessorHost.java:1065) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:306) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1650) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1396) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1962) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1989) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] 2023-08-10T06:23:48,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1532): Rolled back pid=64, state=ROLLEDBACK, exception=java.io.IOException via master-snapshot:java.io.IOException: preSnapshot; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } exec-time=22 msec 2023-08-10T06:23:48,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-08-10T06:23:48,536 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60c8f7c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:23:48,542 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f47ab23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:23:48,542 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:23:48,544 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:38034, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:23:48,548 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:23:48,550 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56368, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService REPAIR status: no failed sessions found. Checking failed delete backup operation ... Found failed DELETE operation for: backup_1691648616770 Running DELETE again ... 2023-08-10T06:23:48,564 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-08-10T06:23:48,569 WARN [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1530): Could not restore backup system table. Snapshot snapshot_backup_system does not exists. 2023-08-10T06:23:48,569 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:23:48,583 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:23:48,594 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:23:48,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648628594 (current time:1691648628594). 2023-08-10T06:23:48,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:23:48,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:23:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31c1148a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:23:48,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:23:48,607 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56382, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:23:48,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:23:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:23:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure2.ProcedureExecutor(1033): Stored pid=65, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:23:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 65 2023-08-10T06:23:48,610 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:23:48,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=65 2023-08-10T06:23:48,611 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:23:48,613 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:23:48,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=65 2023-08-10T06:23:48,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=65 2023-08-10T06:23:49,022 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:23:49,023 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE; SnapshotRegionProcedure 130db0832f74edda61b4f19f117dbb07}] 2023-08-10T06:23:49,025 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=66, ppid=65, state=RUNNABLE; SnapshotRegionProcedure 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:49,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37733 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=66 2023-08-10T06:23:49,177 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:23:49,177 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.HRegion(2819): Flushing 130db0832f74edda61b4f19f117dbb07 2/2 column families, dataSize=1.21 KB heapSize=2.59 KB 2023-08-10T06:23:49,186 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/meta/a4932b46610847bfa479b820bdd0cba7 is 169, key is trslm:hdfs://localhost:46823/backupUT\x00test-1691648567409/meta:log-roll-map/1691648627912/Put/seqid=0 2023-08-10T06:23:49,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=65 2023-08-10T06:23:49,594 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=571 B at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/meta/a4932b46610847bfa479b820bdd0cba7 2023-08-10T06:23:49,601 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a4932b46610847bfa479b820bdd0cba7 2023-08-10T06:23:49,605 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/session/6bdccac655a2459db6a39dd875fd0516 is 310, key is session:backup_1691648616770/session:context/1691648628339/Put/seqid=0 2023-08-10T06:23:49,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=65 2023-08-10T06:23:50,015 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=668 B at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/session/6bdccac655a2459db6a39dd875fd0516 2023-08-10T06:23:50,021 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6bdccac655a2459db6a39dd875fd0516 2023-08-10T06:23:50,022 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/meta/a4932b46610847bfa479b820bdd0cba7 as hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/a4932b46610847bfa479b820bdd0cba7 2023-08-10T06:23:50,027 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/a4932b46610847bfa479b820bdd0cba7, entries=5, sequenceid=80, filesize=5.5 K 2023-08-10T06:23:50,027 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/session/6bdccac655a2459db6a39dd875fd0516 as hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/6bdccac655a2459db6a39dd875fd0516 2023-08-10T06:23:50,032 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/6bdccac655a2459db6a39dd875fd0516, entries=2, sequenceid=80, filesize=5.3 K 2023-08-10T06:23:50,033 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.HRegion(3022): Finished flush of dataSize ~1.21 KB/1239, heapSize ~2.55 KB/2616, currentSize=0 B/0 for 130db0832f74edda61b4f19f117dbb07 in 856ms, sequenceid=80, compaction requested=true 2023-08-10T06:23:50,033 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.HRegion(2520): Flush status journal for 130db0832f74edda61b4f19f117dbb07: 2023-08-10T06:23:50,033 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. for snapshot_backup_system completed. 2023-08-10T06:23:50,033 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:23:50,033 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:23:50,033 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/6d6621fec99f4115886e4e523164d414, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/1e7ba44d3ce0479a82c66ad8d16173e2, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/bc7b88341c1f4449b337245ba1560753, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/287d80b85c894371951b5e08d8d1fa60, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/a4932b46610847bfa479b820bdd0cba7] hfiles 2023-08-10T06:23:50,033 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/6d6621fec99f4115886e4e523164d414 for snapshot=snapshot_backup_system 2023-08-10T06:23:50,033 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/1e7ba44d3ce0479a82c66ad8d16173e2 for snapshot=snapshot_backup_system 2023-08-10T06:23:50,034 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/bc7b88341c1f4449b337245ba1560753 for snapshot=snapshot_backup_system 2023-08-10T06:23:50,034 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/287d80b85c894371951b5e08d8d1fa60 for snapshot=snapshot_backup_system 2023-08-10T06:23:50,034 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/a4932b46610847bfa479b820bdd0cba7 for snapshot=snapshot_backup_system 2023-08-10T06:23:50,034 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/840b366be1f149a6a0f25329a1515460, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/6f9e71fc2fea41d49c5fddcb29e166d7, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/a066d9ae88754eafb7320b716301a931, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/ab1bb1e7cf6d4ca083d6ddaae454a35d, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/3215b68d03e9468cbb851950cf65b0d4, hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/6bdccac655a2459db6a39dd875fd0516] hfiles 2023-08-10T06:23:50,034 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (1/6): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/840b366be1f149a6a0f25329a1515460 for snapshot=snapshot_backup_system 2023-08-10T06:23:50,034 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (2/6): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/6f9e71fc2fea41d49c5fddcb29e166d7 for snapshot=snapshot_backup_system 2023-08-10T06:23:50,034 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (3/6): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/a066d9ae88754eafb7320b716301a931 for snapshot=snapshot_backup_system 2023-08-10T06:23:50,034 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (4/6): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/ab1bb1e7cf6d4ca083d6ddaae454a35d for snapshot=snapshot_backup_system 2023-08-10T06:23:50,034 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (5/6): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/3215b68d03e9468cbb851950cf65b0d4 for snapshot=snapshot_backup_system 2023-08-10T06:23:50,034 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (6/6): hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/6bdccac655a2459db6a39dd875fd0516 for snapshot=snapshot_backup_system 2023-08-10T06:23:50,439 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:23:50,440 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=66 2023-08-10T06:23:50,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.HMaster(3994): Remote procedure done, pid=66 2023-08-10T06:23:50,440 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:50,442 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=66, ppid=65, state=RUNNABLE; SnapshotRegionProcedure 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:50,445 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=66, resume processing ppid=65 2023-08-10T06:23:50,445 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:23:50,446 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=66, ppid=65, state=SUCCESS; SnapshotRegionProcedure 130db0832f74edda61b4f19f117dbb07 in 1.4200 sec 2023-08-10T06:23:50,446 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:23:50,447 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:23:50,447 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:23:50,448 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:23:50,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=65 2023-08-10T06:23:50,858 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:23:50,869 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:23:50,870 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:23:51,688 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:23:51,688 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 65 2023-08-10T06:23:51,690 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=65, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0800 sec 2023-08-10T06:23:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=65 2023-08-10T06:23:52,724 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupAdminImpl(227): Deleting backup backup_1691648616770 ... 2023-08-10T06:23:52,725 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] util.BackupUtils(435): Delete log files: .backup.manifest 2023-08-10T06:23:52,726 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] util.BackupUtils(435): Delete log files: default 2023-08-10T06:23:52,726 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] util.BackupUtils(443): Trying to cleanup up target dir : backup_1691648616770 2023-08-10T06:23:52,727 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] util.BackupUtils(458): No data has been found in hdfs://localhost:46823/backupUT/backup_1691648616770/default/test-1691648567409. 2023-08-10T06:23:52,728 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] util.BackupUtils(465): hdfs://localhost:46823/backupUT/backup_1691648616770/default is empty, remove it. 2023-08-10T06:23:52,739 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupAdminImpl(269): 0 bulk loaded files out of 0 were deleted 2023-08-10T06:23:52,739 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(433): deleted 0 original bulkload rows 2023-08-10T06:23:52,741 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupAdminImpl(276): Delete backup backup_1691648616770 completed. 2023-08-10T06:23:52,751 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1014): test-1691648567409 2023-08-10T06:23:52,754 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:23:52,763 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:23:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:23:52,766 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:23:52,766 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(637): Finish backup exclusive operation DELETE operation finished OK: backup_1691648616770 No failed backup MERGE operation found 2023-08-10T06:23:52,770 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:23:52,774 ERROR [Listener at jenkins-hbase3.apache.org/36469 {}] impl.BackupSystemTable(1559): Snapshot snapshot_backup_system does not exists 2023-08-10T06:23:52,775 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:23:52,775 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:23:52,799 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] hbase.ResourceChecker(175): after: backup.TestBackupDeleteWithFailures#testBackupDeleteWithFailures Thread=593 (was 583) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: member: 'jenkins-hbase3.apache.org,37733,1691648558732' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1277506431-172.31.12.81-1691648552430:blk_1073741924_1100, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: region-location-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: (jenkins-hbase3.apache.org,39523,1691648555675)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_975510334_16 at /127.0.0.1:52610 [Waiting for operation #28] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1293226095_16 at /127.0.0.1:36816 [Waiting for operation #14] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1277506431-172.31.12.81-1691648552430:blk_1073741923_1099, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:61601@0x7c89e4a5-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/cluster_9839947b-fd0a-a12a-305c-b4c05ada1651/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,37733,1691648558732)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:61601@0x7c89e4a5-SendThread(127.0.0.1:61601) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1261) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:61601@0x2a9ccb8f java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1455) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1485) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$488/1592305185.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_975510334_16 at /127.0.0.1:57542 [Waiting for operation #14] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:61601@0x7c89e4a5 java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1455) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1485) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$488/1592305185.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/cluster_9839947b-fd0a-a12a-305c-b4c05ada1651/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,37733,1691648558732)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:61601@0x2a9ccb8f-SendThread(127.0.0.1:61601) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:228) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1338) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1261) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-190070831_16 at /127.0.0.1:57494 [Receiving block BP-1277506431-172.31.12.81-1691648552430:blk_1073741924_1100] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,37733,1691648558732)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-190070831_16 at /127.0.0.1:57482 [Receiving block BP-1277506431-172.31.12.81-1691648552430:blk_1073741923_1099] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:61601@0x2a9ccb8f-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: hconnection-0x761a7a37-shared-pool-31 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=680 (was 660) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=111 (was 209), ProcessCount=173 (was 173), AvailableMemoryMB=2176 (was 2333) 2023-08-10T06:23:52,800 WARN [Listener at jenkins-hbase3.apache.org/36469 {}] hbase.ResourceChecker(130): Thread=593 is superior to 500 2023-08-10T06:23:52,809 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1691648592646" 2023-08-10T06:23:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39523 {}] snapshot.SnapshotManager(378): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1691648592646 2023-08-10T06:23:52,814 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-08-10T06:23:52,814 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:23:52,814 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-08-10T06:23:52,814 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] util.JVMClusterUtil(257): Found active master hash=252242423, stopped=false 2023-08-10T06:23:52,815 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver 2023-08-10T06:23:52,815 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] master.ServerManager(910): Cluster shutdown requested of master=jenkins-hbase3.apache.org,39523,1691648555675 2023-08-10T06:23:52,823 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] procedure2.ProcedureExecutor(631): Stopping 2023-08-10T06:23:52,824 DEBUG [Listener at jenkins-hbase3.apache.org/36469 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:23:52,824 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] regionserver.HRegionServer(2513): ***** STOPPING region server 'jenkins-hbase3.apache.org,37733,1691648558732' ***** 2023-08-10T06:23:52,824 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] regionserver.HRegionServer(2527): STOPPED: Shutdown requested 2023-08-10T06:23:52,825 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-08-10T06:23:52,842 INFO [RS:0;jenkins-hbase3:37733 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@578183c5{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-08-10T06:23:52,847 INFO [RS:0;jenkins-hbase3:37733 {}] server.AbstractConnector(383): Stopped ServerConnector@52b0ba37{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-08-10T06:23:52,847 INFO [RS:0;jenkins-hbase3:37733 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-08-10T06:23:52,848 INFO [RS:0;jenkins-hbase3:37733 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@53f1d32b{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-08-10T06:23:52,849 INFO [RS:0;jenkins-hbase3:37733 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@14e73191{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/hadoop.log.dir/,STOPPED} 2023-08-10T06:23:52,851 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.HeapMemoryManager(220): Stopping 2023-08-10T06:23:52,851 INFO [RS:0;jenkins-hbase3:37733 {}] flush.RegionServerFlushTableProcedureManager(117): Stopping region server flush procedure manager gracefully. 2023-08-10T06:23:52,852 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-08-10T06:23:52,852 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-08-10T06:23:52,852 INFO [RS:0;jenkins-hbase3:37733 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-08-10T06:23:52,852 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(3524): Received CLOSE for 186ba7be42f319c9f99e9a75814a2785 2023-08-10T06:23:52,853 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(3524): Received CLOSE for 47fa27f3f64d1137e3edbb66cac8a6d3 2023-08-10T06:23:52,853 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(3524): Received CLOSE for b203bddb45f38798df364fa4839b7781 2023-08-10T06:23:52,853 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(3524): Received CLOSE for 39c91a27ba70e64360f6a7e105d4cca0 2023-08-10T06:23:52,853 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(3524): Received CLOSE for aebeeaff0240a9e4a35acc504f682275 2023-08-10T06:23:52,853 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(3524): Received CLOSE for 130db0832f74edda61b4f19f117dbb07 2023-08-10T06:23:52,853 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(3524): Received CLOSE for 89ac3e8e753d2c1222fc1445a7518dbe 2023-08-10T06:23:52,854 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,37733,1691648558732 2023-08-10T06:23:52,854 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-08-10T06:23:52,854 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 186ba7be42f319c9f99e9a75814a2785, disabling compactions & flushes 2023-08-10T06:23:52,854 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-16916485674092,,1691648570371.186ba7be42f319c9f99e9a75814a2785. 2023-08-10T06:23:52,854 DEBUG [RS:0;jenkins-hbase3:37733 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:23:52,854 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16916485674092,,1691648570371.186ba7be42f319c9f99e9a75814a2785. 2023-08-10T06:23:52,854 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16916485674092,,1691648570371.186ba7be42f319c9f99e9a75814a2785. after waiting 0 ms 2023-08-10T06:23:52,854 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16916485674092,,1691648570371.186ba7be42f319c9f99e9a75814a2785. 2023-08-10T06:23:52,854 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-08-10T06:23:52,854 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-08-10T06:23:52,854 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-08-10T06:23:52,854 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(3524): Received CLOSE for 1588230740 2023-08-10T06:23:52,854 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(1579): Waiting on 8 regions to close 2023-08-10T06:23:52,854 DEBUG [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(1583): Online Regions={186ba7be42f319c9f99e9a75814a2785=ns3:test-16916485674092,,1691648570371.186ba7be42f319c9f99e9a75814a2785., 47fa27f3f64d1137e3edbb66cac8a6d3=ns4:test-16916485674093,,1691648572496.47fa27f3f64d1137e3edbb66cac8a6d3., b203bddb45f38798df364fa4839b7781=ns2:test-16916485674091,,1691648569165.b203bddb45f38798df364fa4839b7781., 1588230740=hbase:meta,,1.1588230740, 39c91a27ba70e64360f6a7e105d4cca0=hbase:namespace,,1691648561443.39c91a27ba70e64360f6a7e105d4cca0., aebeeaff0240a9e4a35acc504f682275=backup:system_bulk,,1691648576005.aebeeaff0240a9e4a35acc504f682275., 130db0832f74edda61b4f19f117dbb07=backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07., 89ac3e8e753d2c1222fc1445a7518dbe=test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe.} 2023-08-10T06:23:52,855 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-08-10T06:23:52,855 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-08-10T06:23:52,855 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-08-10T06:23:52,856 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-08-10T06:23:52,856 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-08-10T06:23:52,856 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=13.22 KB heapSize=23.11 KB 2023-08-10T06:23:52,856 DEBUG [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(1609): Waiting on 130db0832f74edda61b4f19f117dbb07, 1588230740, 186ba7be42f319c9f99e9a75814a2785, 39c91a27ba70e64360f6a7e105d4cca0, 47fa27f3f64d1137e3edbb66cac8a6d3, 89ac3e8e753d2c1222fc1445a7518dbe, aebeeaff0240a9e4a35acc504f682275, b203bddb45f38798df364fa4839b7781 2023-08-10T06:23:52,862 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/ns3/test-16916485674092/186ba7be42f319c9f99e9a75814a2785/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-08-10T06:23:52,863 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:23:52,864 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-16916485674092,,1691648570371.186ba7be42f319c9f99e9a75814a2785. 2023-08-10T06:23:52,864 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 186ba7be42f319c9f99e9a75814a2785: 2023-08-10T06:23:52,865 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-16916485674092,,1691648570371.186ba7be42f319c9f99e9a75814a2785. 2023-08-10T06:23:52,865 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 47fa27f3f64d1137e3edbb66cac8a6d3, disabling compactions & flushes 2023-08-10T06:23:52,866 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-16916485674093,,1691648572496.47fa27f3f64d1137e3edbb66cac8a6d3. 2023-08-10T06:23:52,866 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16916485674093,,1691648572496.47fa27f3f64d1137e3edbb66cac8a6d3. 2023-08-10T06:23:52,866 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16916485674093,,1691648572496.47fa27f3f64d1137e3edbb66cac8a6d3. after waiting 0 ms 2023-08-10T06:23:52,866 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16916485674093,,1691648572496.47fa27f3f64d1137e3edbb66cac8a6d3. 2023-08-10T06:23:52,874 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/ns4/test-16916485674093/47fa27f3f64d1137e3edbb66cac8a6d3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-08-10T06:23:52,875 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:23:52,877 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-16916485674093,,1691648572496.47fa27f3f64d1137e3edbb66cac8a6d3. 2023-08-10T06:23:52,877 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 47fa27f3f64d1137e3edbb66cac8a6d3: 2023-08-10T06:23:52,877 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-16916485674093,,1691648572496.47fa27f3f64d1137e3edbb66cac8a6d3. 2023-08-10T06:23:52,878 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing b203bddb45f38798df364fa4839b7781, disabling compactions & flushes 2023-08-10T06:23:52,878 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-16916485674091,,1691648569165.b203bddb45f38798df364fa4839b7781. 2023-08-10T06:23:52,878 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16916485674091,,1691648569165.b203bddb45f38798df364fa4839b7781. 2023-08-10T06:23:52,878 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16916485674091,,1691648569165.b203bddb45f38798df364fa4839b7781. after waiting 0 ms 2023-08-10T06:23:52,878 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16916485674091,,1691648569165.b203bddb45f38798df364fa4839b7781. 2023-08-10T06:23:52,878 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing b203bddb45f38798df364fa4839b7781 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-08-10T06:23:52,894 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/hbase/meta/1588230740/.tmp/info/0390dee8c9b04f8683638b5a437629cb is 159, key is ns2:test-16916485674091,,1691648569165.b203bddb45f38798df364fa4839b7781./info:regioninfo/1691648569979/Put/seqid=0 2023-08-10T06:23:52,899 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/ns2/test-16916485674091/b203bddb45f38798df364fa4839b7781/.tmp/f/287fb082afe84d0887b272a942cef282 is 37, key is row10/f:q1/1691648570294/Put/seqid=0 2023-08-10T06:23:52,913 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/ns2/test-16916485674091/b203bddb45f38798df364fa4839b7781/.tmp/f/287fb082afe84d0887b272a942cef282 2023-08-10T06:23:52,918 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 287fb082afe84d0887b272a942cef282 2023-08-10T06:23:52,919 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/ns2/test-16916485674091/b203bddb45f38798df364fa4839b7781/.tmp/f/287fb082afe84d0887b272a942cef282 as hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/ns2/test-16916485674091/b203bddb45f38798df364fa4839b7781/f/287fb082afe84d0887b272a942cef282 2023-08-10T06:23:52,924 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/ns2/test-16916485674091/b203bddb45f38798df364fa4839b7781/f/287fb082afe84d0887b272a942cef282, entries=99, sequenceid=103, filesize=8.2 K 2023-08-10T06:23:52,925 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for b203bddb45f38798df364fa4839b7781 in 47ms, sequenceid=103, compaction requested=false 2023-08-10T06:23:52,928 DEBUG [StoreFileCloser-b203bddb45f38798df364fa4839b7781-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 287fb082afe84d0887b272a942cef282 2023-08-10T06:23:52,933 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/ns2/test-16916485674091/b203bddb45f38798df364fa4839b7781/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-08-10T06:23:52,933 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:23:52,934 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-16916485674091,,1691648569165.b203bddb45f38798df364fa4839b7781. 2023-08-10T06:23:52,934 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for b203bddb45f38798df364fa4839b7781: 2023-08-10T06:23:52,934 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-16916485674091,,1691648569165.b203bddb45f38798df364fa4839b7781. 2023-08-10T06:23:52,935 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 39c91a27ba70e64360f6a7e105d4cca0, disabling compactions & flushes 2023-08-10T06:23:52,935 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1691648561443.39c91a27ba70e64360f6a7e105d4cca0. 2023-08-10T06:23:52,935 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1691648561443.39c91a27ba70e64360f6a7e105d4cca0. 2023-08-10T06:23:52,935 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1691648561443.39c91a27ba70e64360f6a7e105d4cca0. after waiting 0 ms 2023-08-10T06:23:52,935 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1691648561443.39c91a27ba70e64360f6a7e105d4cca0. 2023-08-10T06:23:52,935 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 39c91a27ba70e64360f6a7e105d4cca0 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-08-10T06:23:52,950 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/hbase/namespace/39c91a27ba70e64360f6a7e105d4cca0/.tmp/info/97a891c4a4534ff097b8a9e2ff559620 is 45, key is default/info:d/1691648562429/Put/seqid=0 2023-08-10T06:23:53,056 DEBUG [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(1609): Waiting on 130db0832f74edda61b4f19f117dbb07, 1588230740, 39c91a27ba70e64360f6a7e105d4cca0, 89ac3e8e753d2c1222fc1445a7518dbe, aebeeaff0240a9e4a35acc504f682275 2023-08-10T06:23:53,257 DEBUG [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(1609): Waiting on 130db0832f74edda61b4f19f117dbb07, 1588230740, 39c91a27ba70e64360f6a7e105d4cca0, 89ac3e8e753d2c1222fc1445a7518dbe, aebeeaff0240a9e4a35acc504f682275 2023-08-10T06:23:53,312 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.18 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/hbase/meta/1588230740/.tmp/info/0390dee8c9b04f8683638b5a437629cb 2023-08-10T06:23:53,320 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 0390dee8c9b04f8683638b5a437629cb 2023-08-10T06:23:53,339 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/hbase/meta/1588230740/.tmp/table/a7d576ca9bf14ce798a37d9238ad95c5 is 59, key is ns2:test-16916485674091/table:state/1691648569990/Put/seqid=0 2023-08-10T06:23:53,355 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/hbase/namespace/39c91a27ba70e64360f6a7e105d4cca0/.tmp/info/97a891c4a4534ff097b8a9e2ff559620 2023-08-10T06:23:53,361 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 97a891c4a4534ff097b8a9e2ff559620 2023-08-10T06:23:53,362 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/hbase/namespace/39c91a27ba70e64360f6a7e105d4cca0/.tmp/info/97a891c4a4534ff097b8a9e2ff559620 as hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/hbase/namespace/39c91a27ba70e64360f6a7e105d4cca0/info/97a891c4a4534ff097b8a9e2ff559620 2023-08-10T06:23:53,368 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/hbase/namespace/39c91a27ba70e64360f6a7e105d4cca0/info/97a891c4a4534ff097b8a9e2ff559620, entries=7, sequenceid=11, filesize=5.1 K 2023-08-10T06:23:53,369 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 39c91a27ba70e64360f6a7e105d4cca0 in 434ms, sequenceid=11, compaction requested=false 2023-08-10T06:23:53,374 DEBUG [StoreFileCloser-39c91a27ba70e64360f6a7e105d4cca0-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 97a891c4a4534ff097b8a9e2ff559620 2023-08-10T06:23:53,382 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/hbase/namespace/39c91a27ba70e64360f6a7e105d4cca0/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-08-10T06:23:53,383 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:23:53,384 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1691648561443.39c91a27ba70e64360f6a7e105d4cca0. 2023-08-10T06:23:53,384 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 39c91a27ba70e64360f6a7e105d4cca0: 2023-08-10T06:23:53,384 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1691648561443.39c91a27ba70e64360f6a7e105d4cca0. 2023-08-10T06:23:53,385 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing aebeeaff0240a9e4a35acc504f682275, disabling compactions & flushes 2023-08-10T06:23:53,385 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1691648576005.aebeeaff0240a9e4a35acc504f682275. 2023-08-10T06:23:53,385 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1691648576005.aebeeaff0240a9e4a35acc504f682275. 2023-08-10T06:23:53,385 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1691648576005.aebeeaff0240a9e4a35acc504f682275. after waiting 0 ms 2023-08-10T06:23:53,385 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1691648576005.aebeeaff0240a9e4a35acc504f682275. 2023-08-10T06:23:53,390 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system_bulk/aebeeaff0240a9e4a35acc504f682275/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-08-10T06:23:53,391 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:23:53,392 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1691648576005.aebeeaff0240a9e4a35acc504f682275. 2023-08-10T06:23:53,392 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for aebeeaff0240a9e4a35acc504f682275: 2023-08-10T06:23:53,392 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1691648576005.aebeeaff0240a9e4a35acc504f682275. 2023-08-10T06:23:53,392 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 130db0832f74edda61b4f19f117dbb07, disabling compactions & flushes 2023-08-10T06:23:53,392 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:23:53,392 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:23:53,392 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. after waiting 0 ms 2023-08-10T06:23:53,392 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:23:53,392 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 130db0832f74edda61b4f19f117dbb07 2/2 column families, dataSize=223 B heapSize=1.01 KB 2023-08-10T06:23:53,397 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/meta/ada40083c98946b89089cbef661c22bf is 91, key is incrbackupset:hdfs://localhost:46823/backupUT/meta:test-1691648567409/1691648632752/Put/seqid=0 2023-08-10T06:23:53,457 DEBUG [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(1609): Waiting on 130db0832f74edda61b4f19f117dbb07, 1588230740, 89ac3e8e753d2c1222fc1445a7518dbe 2023-08-10T06:23:53,554 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-08-10T06:23:53,554 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-08-10T06:23:53,657 DEBUG [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(1609): Waiting on 130db0832f74edda61b4f19f117dbb07, 1588230740, 89ac3e8e753d2c1222fc1445a7518dbe 2023-08-10T06:23:53,746 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/hbase/meta/1588230740/.tmp/table/a7d576ca9bf14ce798a37d9238ad95c5 2023-08-10T06:23:53,752 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a7d576ca9bf14ce798a37d9238ad95c5 2023-08-10T06:23:53,752 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/hbase/meta/1588230740/.tmp/info/0390dee8c9b04f8683638b5a437629cb as hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/hbase/meta/1588230740/info/0390dee8c9b04f8683638b5a437629cb 2023-08-10T06:23:53,757 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/hbase/meta/1588230740/info/0390dee8c9b04f8683638b5a437629cb, entries=72, sequenceid=59, filesize=14.3 K 2023-08-10T06:23:53,759 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/hbase/meta/1588230740/.tmp/table/a7d576ca9bf14ce798a37d9238ad95c5 as hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/hbase/meta/1588230740/table/a7d576ca9bf14ce798a37d9238ad95c5 2023-08-10T06:23:53,764 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/hbase/meta/1588230740/table/a7d576ca9bf14ce798a37d9238ad95c5, entries=15, sequenceid=59, filesize=5.8 K 2023-08-10T06:23:53,765 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~13.22 KB/13542, heapSize ~22.83 KB/23376, currentSize=0 B/0 for 1588230740 in 909ms, sequenceid=59, compaction requested=false 2023-08-10T06:23:53,767 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 0390dee8c9b04f8683638b5a437629cb 2023-08-10T06:23:53,769 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a7d576ca9bf14ce798a37d9238ad95c5 2023-08-10T06:23:53,773 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/hbase/meta/1588230740/recovered.edits/62.seqid, newMaxSeqId=62, maxSeqId=1 2023-08-10T06:23:53,774 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:23:53,774 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-08-10T06:23:53,775 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-08-10T06:23:53,775 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-08-10T06:23:53,775 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-08-10T06:23:53,804 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=124 B at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/meta/ada40083c98946b89089cbef661c22bf 2023-08-10T06:23:53,809 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ada40083c98946b89089cbef661c22bf 2023-08-10T06:23:53,809 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: ada40083c98946b89089cbef661c22bf 2023-08-10T06:23:53,812 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/session/911782e5706e43748f94265d5d8cbaed is 59, key is session:backup_1691648616770/session:/1691648632740/DeleteFamily/seqid=0 2023-08-10T06:23:53,857 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(1579): Waiting on 2 regions to close 2023-08-10T06:23:53,857 DEBUG [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(1583): Online Regions={130db0832f74edda61b4f19f117dbb07=backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07., 89ac3e8e753d2c1222fc1445a7518dbe=test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe.} 2023-08-10T06:23:53,858 DEBUG [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(1609): Waiting on 130db0832f74edda61b4f19f117dbb07, 89ac3e8e753d2c1222fc1445a7518dbe 2023-08-10T06:23:54,058 DEBUG [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(1609): Waiting on 130db0832f74edda61b4f19f117dbb07, 89ac3e8e753d2c1222fc1445a7518dbe 2023-08-10T06:23:54,217 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=99 B at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/session/911782e5706e43748f94265d5d8cbaed 2023-08-10T06:23:54,222 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 911782e5706e43748f94265d5d8cbaed 2023-08-10T06:23:54,222 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 911782e5706e43748f94265d5d8cbaed 2023-08-10T06:23:54,223 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/meta/ada40083c98946b89089cbef661c22bf as hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/ada40083c98946b89089cbef661c22bf 2023-08-10T06:23:54,228 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ada40083c98946b89089cbef661c22bf 2023-08-10T06:23:54,229 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/meta/ada40083c98946b89089cbef661c22bf, entries=2, sequenceid=87, filesize=5.2 K 2023-08-10T06:23:54,230 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/.tmp/session/911782e5706e43748f94265d5d8cbaed as hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/911782e5706e43748f94265d5d8cbaed 2023-08-10T06:23:54,235 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 911782e5706e43748f94265d5d8cbaed 2023-08-10T06:23:54,235 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/session/911782e5706e43748f94265d5d8cbaed, entries=2, sequenceid=87, filesize=5.2 K 2023-08-10T06:23:54,236 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~223 B/223, heapSize ~1000 B/1000, currentSize=0 B/0 for 130db0832f74edda61b4f19f117dbb07 in 844ms, sequenceid=87, compaction requested=true 2023-08-10T06:23:54,238 DEBUG [StoreFileCloser-130db0832f74edda61b4f19f117dbb07-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6d6621fec99f4115886e4e523164d414 2023-08-10T06:23:54,240 DEBUG [StoreFileCloser-130db0832f74edda61b4f19f117dbb07-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 1e7ba44d3ce0479a82c66ad8d16173e2 2023-08-10T06:23:54,242 DEBUG [StoreFileCloser-130db0832f74edda61b4f19f117dbb07-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: bc7b88341c1f4449b337245ba1560753 2023-08-10T06:23:54,244 DEBUG [StoreFileCloser-130db0832f74edda61b4f19f117dbb07-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 287d80b85c894371951b5e08d8d1fa60 2023-08-10T06:23:54,246 DEBUG [StoreFileCloser-130db0832f74edda61b4f19f117dbb07-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a4932b46610847bfa479b820bdd0cba7 2023-08-10T06:23:54,247 DEBUG [StoreFileCloser-130db0832f74edda61b4f19f117dbb07-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: ada40083c98946b89089cbef661c22bf 2023-08-10T06:23:54,250 DEBUG [StoreFileCloser-130db0832f74edda61b4f19f117dbb07-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 840b366be1f149a6a0f25329a1515460 2023-08-10T06:23:54,254 DEBUG [StoreFileCloser-130db0832f74edda61b4f19f117dbb07-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6f9e71fc2fea41d49c5fddcb29e166d7 2023-08-10T06:23:54,254 DEBUG [StoreFileCloser-130db0832f74edda61b4f19f117dbb07-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a066d9ae88754eafb7320b716301a931 2023-08-10T06:23:54,254 DEBUG [StoreFileCloser-130db0832f74edda61b4f19f117dbb07-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: ab1bb1e7cf6d4ca083d6ddaae454a35d 2023-08-10T06:23:54,254 DEBUG [StoreFileCloser-130db0832f74edda61b4f19f117dbb07-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3215b68d03e9468cbb851950cf65b0d4 2023-08-10T06:23:54,254 DEBUG [StoreFileCloser-130db0832f74edda61b4f19f117dbb07-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6bdccac655a2459db6a39dd875fd0516 2023-08-10T06:23:54,254 DEBUG [StoreFileCloser-130db0832f74edda61b4f19f117dbb07-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 911782e5706e43748f94265d5d8cbaed 2023-08-10T06:23:54,258 DEBUG [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(1609): Waiting on 130db0832f74edda61b4f19f117dbb07, 89ac3e8e753d2c1222fc1445a7518dbe 2023-08-10T06:23:54,258 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/backup/system/130db0832f74edda61b4f19f117dbb07/recovered.edits/90.seqid, newMaxSeqId=90, maxSeqId=34 2023-08-10T06:23:54,259 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:23:54,259 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:23:54,259 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 130db0832f74edda61b4f19f117dbb07: 2023-08-10T06:23:54,260 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1691648574866.130db0832f74edda61b4f19f117dbb07. 2023-08-10T06:23:54,260 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 89ac3e8e753d2c1222fc1445a7518dbe, disabling compactions & flushes 2023-08-10T06:23:54,260 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe. 2023-08-10T06:23:54,260 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe. 2023-08-10T06:23:54,260 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe. after waiting 0 ms 2023-08-10T06:23:54,260 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe. 2023-08-10T06:23:54,260 DEBUG [StoreFileCloser-89ac3e8e753d2c1222fc1445a7518dbe-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6ec84d0138b7498790d3e4f4273b9498 2023-08-10T06:23:54,264 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/data/default/test-1691648567409/89ac3e8e753d2c1222fc1445a7518dbe/recovered.edits/108.seqid, newMaxSeqId=108, maxSeqId=1 2023-08-10T06:23:54,264 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:23:54,265 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe. 2023-08-10T06:23:54,265 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 89ac3e8e753d2c1222fc1445a7518dbe: 2023-08-10T06:23:54,266 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1691648567409,,1691648567903.89ac3e8e753d2c1222fc1445a7518dbe. 2023-08-10T06:23:54,458 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,37733,1691648558732; all regions closed. 2023-08-10T06:23:54,471 DEBUG [RS:0;jenkins-hbase3:37733 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/oldWALs 2023-08-10T06:23:54,471 INFO [RS:0;jenkins-hbase3:37733 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C37733%2C1691648558732.meta:.meta(num 1691648620954) 2023-08-10T06:23:54,475 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/WALs/jenkins-hbase3.apache.org,37733,1691648558732/jenkins-hbase3.apache.org%2C37733%2C1691648558732.1691648620933 not finished, retry = 0 2023-08-10T06:23:54,581 DEBUG [RS:0;jenkins-hbase3:37733 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/oldWALs 2023-08-10T06:23:54,582 INFO [RS:0;jenkins-hbase3:37733 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C37733%2C1691648558732:(num 1691648620933) 2023-08-10T06:23:54,582 DEBUG [RS:0;jenkins-hbase3:37733 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:23:54,582 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.LeaseManager(133): Closed leases 2023-08-10T06:23:54,582 INFO [RS:0;jenkins-hbase3:37733 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2023-08-10T06:23:54,583 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-08-10T06:23:54,584 INFO [RS:0;jenkins-hbase3:37733 {}] ipc.NettyRpcServer(213): Stopping server on /172.31.12.81:37733 2023-08-10T06:23:54,596 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,37733,1691648558732] 2023-08-10T06:23:54,596 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,37733,1691648558732; numProcessing=1 2023-08-10T06:23:54,598 INFO [RegionServerTracker-0 {}] master.ServerManager(569): Cluster shutdown set; jenkins-hbase3.apache.org,37733,1691648558732 expired; onlineServers=0 2023-08-10T06:23:54,598 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2513): ***** STOPPING region server 'jenkins-hbase3.apache.org,39523,1691648555675' ***** 2023-08-10T06:23:54,598 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2527): STOPPED: Cluster shutdown set; onlineServer=0 2023-08-10T06:23:54,599 DEBUG [M:0;jenkins-hbase3:39523 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fdfffad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-08-10T06:23:54,599 INFO [M:0;jenkins-hbase3:39523 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-08-10T06:23:54,644 INFO [M:0;jenkins-hbase3:39523 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@255e5c0e{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-08-10T06:23:54,645 INFO [M:0;jenkins-hbase3:39523 {}] server.AbstractConnector(383): Stopped ServerConnector@39508194{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-08-10T06:23:54,645 INFO [M:0;jenkins-hbase3:39523 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-08-10T06:23:54,646 INFO [M:0;jenkins-hbase3:39523 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@44f6082b{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-08-10T06:23:54,647 INFO [M:0;jenkins-hbase3:39523 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@6e84de57{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/hadoop.log.dir/,STOPPED} 2023-08-10T06:23:54,647 INFO [M:0;jenkins-hbase3:39523 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,39523,1691648555675 2023-08-10T06:23:54,647 INFO [M:0;jenkins-hbase3:39523 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,39523,1691648555675; all regions closed. 2023-08-10T06:23:54,647 DEBUG [M:0;jenkins-hbase3:39523 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:23:54,647 INFO [M:0;jenkins-hbase3:39523 {}] master.HMaster(1664): Stopping master jetty server 2023-08-10T06:23:54,648 INFO [M:0;jenkins-hbase3:39523 {}] server.AbstractConnector(383): Stopped ServerConnector@2edbc0d5{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-08-10T06:23:54,649 DEBUG [M:0;jenkins-hbase3:39523 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-08-10T06:23:54,649 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-08-10T06:23:54,649 INFO [M:0;jenkins-hbase3:39523 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-08-10T06:23:54,649 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1691648560242 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1691648560242,5,FailOnTimeoutGroup] 2023-08-10T06:23:54,649 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1691648560240 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1691648560240,5,FailOnTimeoutGroup] 2023-08-10T06:23:54,649 DEBUG [M:0;jenkins-hbase3:39523 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:23:54,649 DEBUG [M:0;jenkins-hbase3:39523 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-08-10T06:23:54,650 INFO [M:0;jenkins-hbase3:39523 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-08-10T06:23:54,650 DEBUG [M:0;jenkins-hbase3:39523 {}] master.HMaster(1687): Stopping service threads 2023-08-10T06:23:54,650 INFO [M:0;jenkins-hbase3:39523 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-08-10T06:23:54,650 ERROR [M:0;jenkins-hbase3:39523 {}] procedure2.ProcedureExecutor(655): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] 2023-08-10T06:23:54,651 INFO [M:0;jenkins-hbase3:39523 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-08-10T06:23:54,651 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-08-10T06:23:54,652 WARN [M:0;jenkins-hbase3:39523 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-08-10T06:23:54,652 INFO [M:0;jenkins-hbase3:39523 {}] assignment.AssignmentManager(382): Stopping assignment manager 2023-08-10T06:23:54,652 INFO [M:0;jenkins-hbase3:39523 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-08-10T06:23:54,653 DEBUG [M:0;jenkins-hbase3:39523 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-08-10T06:23:54,669 INFO [M:0;jenkins-hbase3:39523 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:23:54,669 DEBUG [M:0;jenkins-hbase3:39523 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:23:54,669 DEBUG [M:0;jenkins-hbase3:39523 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-08-10T06:23:54,669 DEBUG [M:0;jenkins-hbase3:39523 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:23:54,669 INFO [M:0;jenkins-hbase3:39523 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=254.63 KB heapSize=306.35 KB 2023-08-10T06:23:54,687 DEBUG [M:0;jenkins-hbase3:39523 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/51e0a8b5fe7e4afdaec77a0fbb0ce3cd is 88, key is hbase:meta,,1/info:sn/1691648560730/Put/seqid=0 2023-08-10T06:23:54,692 INFO [M:0;jenkins-hbase3:39523 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=615 (bloomFilter=true), to=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/51e0a8b5fe7e4afdaec77a0fbb0ce3cd 2023-08-10T06:23:54,695 INFO [RS:0;jenkins-hbase3:37733 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,37733,1691648558732; zookeeper connection closed. 2023-08-10T06:23:54,696 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@21a8f32 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@21a8f32 2023-08-10T06:23:54,696 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-08-10T06:23:54,698 DEBUG [M:0;jenkins-hbase3:39523 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 51e0a8b5fe7e4afdaec77a0fbb0ce3cd 2023-08-10T06:23:54,718 DEBUG [M:0;jenkins-hbase3:39523 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a7bcf0a6529245099fb8005315e50ef1 is 2094, key is \x00\x00\x00\x00\x00\x00\x00@/proc:d/1691648628438/Put/seqid=0 2023-08-10T06:23:55,124 INFO [M:0;jenkins-hbase3:39523 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=254.03 KB at sequenceid=615 (bloomFilter=true), to=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a7bcf0a6529245099fb8005315e50ef1 2023-08-10T06:23:55,132 INFO [M:0;jenkins-hbase3:39523 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a7bcf0a6529245099fb8005315e50ef1 2023-08-10T06:23:55,133 DEBUG [M:0;jenkins-hbase3:39523 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a7bcf0a6529245099fb8005315e50ef1 2023-08-10T06:23:55,147 DEBUG [M:0;jenkins-hbase3:39523 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2fe87e6a4e084a9eb8a8c0223c4a220e is 82, key is jenkins-hbase3.apache.org,37733,1691648558732/rs:state/1691648560383/Put/seqid=0 2023-08-10T06:23:55,158 INFO [M:0;jenkins-hbase3:39523 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=615 (bloomFilter=true), to=hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2fe87e6a4e084a9eb8a8c0223c4a220e 2023-08-10T06:23:55,164 DEBUG [M:0;jenkins-hbase3:39523 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2fe87e6a4e084a9eb8a8c0223c4a220e 2023-08-10T06:23:55,165 DEBUG [M:0;jenkins-hbase3:39523 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/51e0a8b5fe7e4afdaec77a0fbb0ce3cd as hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/51e0a8b5fe7e4afdaec77a0fbb0ce3cd 2023-08-10T06:23:55,171 INFO [M:0;jenkins-hbase3:39523 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/51e0a8b5fe7e4afdaec77a0fbb0ce3cd, entries=8, sequenceid=615, filesize=5.5 K 2023-08-10T06:23:55,172 DEBUG [M:0;jenkins-hbase3:39523 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a7bcf0a6529245099fb8005315e50ef1 as hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a7bcf0a6529245099fb8005315e50ef1 2023-08-10T06:23:55,177 INFO [M:0;jenkins-hbase3:39523 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a7bcf0a6529245099fb8005315e50ef1 2023-08-10T06:23:55,177 INFO [M:0;jenkins-hbase3:39523 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a7bcf0a6529245099fb8005315e50ef1, entries=66, sequenceid=615, filesize=22.2 K 2023-08-10T06:23:55,178 DEBUG [M:0;jenkins-hbase3:39523 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2fe87e6a4e084a9eb8a8c0223c4a220e as hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2fe87e6a4e084a9eb8a8c0223c4a220e 2023-08-10T06:23:55,183 INFO [M:0;jenkins-hbase3:39523 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:46823/user/jenkins/test-data/43d5f175-5457-395a-55f9-e73785202322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2fe87e6a4e084a9eb8a8c0223c4a220e, entries=1, sequenceid=615, filesize=5.1 K 2023-08-10T06:23:55,184 INFO [M:0;jenkins-hbase3:39523 {}] regionserver.HRegion(3022): Finished flush of dataSize ~254.63 KB/260737, heapSize ~306.05 KB/313400, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 515ms, sequenceid=615, compaction requested=false 2023-08-10T06:23:55,186 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 51e0a8b5fe7e4afdaec77a0fbb0ce3cd 2023-08-10T06:23:55,187 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-proc-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a7bcf0a6529245099fb8005315e50ef1 2023-08-10T06:23:55,190 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-rs-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2fe87e6a4e084a9eb8a8c0223c4a220e 2023-08-10T06:23:55,191 INFO [M:0;jenkins-hbase3:39523 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:23:55,191 DEBUG [M:0;jenkins-hbase3:39523 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-08-10T06:23:55,194 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-08-10T06:23:55,194 INFO [M:0;jenkins-hbase3:39523 {}] flush.MasterFlushTableProcedureManager(81): stop: server shutting down. 2023-08-10T06:23:55,194 INFO [M:0;jenkins-hbase3:39523 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-08-10T06:23:55,195 INFO [M:0;jenkins-hbase3:39523 {}] ipc.NettyRpcServer(213): Stopping server on /172.31.12.81:39523 2023-08-10T06:23:55,298 INFO [M:0;jenkins-hbase3:39523 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,39523,1691648555675; zookeeper connection closed. 2023-08-10T06:23:55,301 WARN [Listener at jenkins-hbase3.apache.org/36469 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-08-10T06:23:55,304 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-08-10T06:23:55,413 WARN [BP-1277506431-172.31.12.81-1691648552430 heartbeating to localhost/127.0.0.1:46823 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-08-10T06:23:55,413 WARN [BP-1277506431-172.31.12.81-1691648552430 heartbeating to localhost/127.0.0.1:46823 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-1277506431-172.31.12.81-1691648552430 (Datanode Uuid 52dea3b8-2a59-4572-b9e0-ede2c541b8c9) service to localhost/127.0.0.1:46823 2023-08-10T06:23:55,415 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/cluster_9839947b-fd0a-a12a-305c-b4c05ada1651/dfs/data/data1/current/BP-1277506431-172.31.12.81-1691648552430 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-08-10T06:23:55,415 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/52317840-1bad-ae43-051f-89cf8ba9d848/cluster_9839947b-fd0a-a12a-305c-b4c05ada1651/dfs/data/data2/current/BP-1277506431-172.31.12.81-1691648552430 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-08-10T06:23:55,471 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-08-10T06:23:55,639 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-08-10T06:23:55,639 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-08-10T06:23:55,651 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-08-10T06:23:55,772 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-08-10T06:23:55,864 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-08-10T06:23:55,864 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-08-10T06:23:55,866 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-08-10T06:23:55,867 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-08-10T06:23:55,867 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-08-10T06:23:55,868 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-08-10T06:23:55,869 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16916485674091 2023-08-10T06:23:55,870 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1691648567409 2023-08-10T06:23:55,871 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16916485674092 2023-08-10T06:23:55,871 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16916485674093 2023-08-10T06:23:55,872 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: Master,sub=Coprocessor.Master.CP_org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver 2023-08-10T06:23:55,872 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:23:55,884 ERROR [Thread[Thread-275,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-08-10T06:23:55,884 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-08-10T06:23:55,992 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-08-10T06:23:56,001 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-08-10T06:23:56,002 ERROR [Thread[Thread-285,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-08-10T06:23:56,015 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-08-10T06:23:56,119 ERROR [Thread[Thread-250,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-08-10T06:23:56,119 INFO [Listener at jenkins-hbase3.apache.org/36469 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped